From d3def3ec9453f6a685bb9390937b3aaae8296f85 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Mon, 1 Aug 2022 11:24:48 +0200 Subject: [PATCH] better user messages, linter errors, refactoring --- firewall/master.go | 10 +++++----- go.mod | 2 +- go.sum | 2 ++ intel/customlists/config.go | 37 ++++++++++++++----------------------- intel/customlists/lists.go | 33 +++++++++++++++++---------------- intel/customlists/module.go | 24 ++++++++++-------------- 6 files changed, 49 insertions(+), 59 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index 50d64bc0..9ba97ced 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -619,7 +619,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the domain name appears in the custom filter list (check for subdomains if enabled) if conn.Entity.Domain != "" { if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok { - conn.Deny(fmt.Sprintf("domain %s matched %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("domain %s matches %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -628,7 +628,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi if p.FilterCNAMEs() { for _, cname := range conn.Entity.CNAME { if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok { - conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matched %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matches %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -637,7 +637,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { if customlists.LookupIP(conn.Entity.IP) { - conn.Deny(fmt.Sprintf("IP address %s appears in the custom filter list", conn.Entity.IP), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("IP address is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } @@ -645,7 +645,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block autonomous system by its number if it appears in the custom filter list if conn.Entity.ASN != 0 { if customlists.LookupASN(conn.Entity.ASN) { - conn.Deny(fmt.Sprintf("autonomous system with number %d appears in the custom filter list", conn.Entity.ASN), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("AS is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } @@ -653,7 +653,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the country appears in the custom filter list if conn.Entity.Country != "" { if customlists.LookupCountry(conn.Entity.Country) { - conn.Deny(fmt.Sprintf("country code %s appears in the custom filter list", conn.Entity.Country), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("country is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } diff --git a/go.mod b/go.mod index 45638ee2..36763c35 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/miekg/dns v1.1.50 github.com/oschwald/maxminddb-golang v1.9.0 - github.com/safing/portbase v0.14.5 + github.com/safing/portbase v0.14.6 github.com/safing/spn v0.4.13 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 0b6989af..0042de61 100644 --- a/go.sum +++ b/go.sum @@ -831,6 +831,8 @@ github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc= github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= +github.com/safing/portbase v0.14.6 h1:AKj1TVJmqOrjXXWcPetchQhHVBwfD3/QDvgmkUj2FxY= +github.com/safing/portbase v0.14.6/go.mod h1:hC6KV9oZsD7yuAom42L5wgOEZ0yAjr2i1r2F5duBSA8= github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s= github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw= github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw= diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 43c0cdd1..a50c4255 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -14,34 +14,25 @@ var ( var getFilePath config.StringOption func registerConfig() error { - help := `The file should contain list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. -Lines that start with a '#' symbol are ignored. -Everything after the first space/tab is ignored. -Example: -############# -\# Domains: -example.com -google.com - -\# IP addresses -1.2.3.4 -4.3.2.1 - -\# Countries -AU -BG - -\# Autonomous Systems -AS123 -############# -> * All the records are stored in RAM, careful with large block lists. -> * Hosts files are not supported.` + help := `The file is checked every couple minutes and will be automatically reloaded when it has changed. + +Entries may be one of: +- Domain: "example.com" +- IP Address: "10.0.0.1" +- Country Code (based on IP): "US" +- AS (Autonomous System): "AS1234" + +Everything after the first element of a line, comments starting with a '#', and empty lines are ignored. +The settings "Block Subdomains of Filter List Entries" and "Block Domain Aliases" also apply to the custom filter list. +Lists in the "Hosts" format are not supported. + +Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.` // register a setting for the file path in the ui err := config.Register(&config.Option{ Name: "Custom Filter List", Key: CfgOptionCustomListBlockingKey, - Description: "Path to the file that contains a list of Domain, IP addresses, country codes and autonomous systems that you want to block", + Description: "Specify the file path to a custom filter list, which will be automatically refreshed. Any connections matching a domain, IP address, Country or ASN in the file will be blocked.", Help: help, OptType: config.OptTypeString, ExpertiseLevel: config.ExpertiseLevelExpert, diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 8d47aa0c..8b4ad7a2 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -2,7 +2,6 @@ package customlists import ( "bufio" - "bytes" "fmt" "net" "os" @@ -10,7 +9,7 @@ import ( "strings" "github.com/miekg/dns" - "github.com/safing/portbase/log" + "github.com/safing/portbase/log" //nolint // weird error "Expected '\n', Found '\t'" "github.com/safing/portbase/notifications" "github.com/safing/portmaster/network/netutils" ) @@ -25,6 +24,7 @@ var ( const ( rationForInvalidLinesUntilWarning = 0.1 parseStatusNotificationID = "customlists:parse-status" + parseWarningNotificationID = "customlists:parse-warning" zeroIPNotificationID = "customlists:too-many-zero-ips" ) @@ -58,8 +58,8 @@ func parseFile(filePath string) error { // open the file if possible file, err := os.Open(filePath) if err != nil { - log.Warningf("intel/customlists: failed to parse file %q ", err) - module.Warning(parseStatusNotificationID, "Failed to open custom filter list", err.Error()) + log.Warningf("intel/customlists: failed to parse file %s", err) + module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error()) return err } defer func() { _ = file.Close() }() @@ -83,33 +83,34 @@ func parseFile(filePath string) error { return err } - var invalidLinesRation float32 = float32(invalidLinesCount) / float32(allLinesCount) + invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount) if invalidLinesRation > rationForInvalidLinesUntilWarning { log.Warning("intel/customlists: Too many invalid lines") - module.Warning(zeroIPNotificationID, "Check your custom filter list, there is too many invalid lines", - fmt.Sprintf(`There are %d from total %d lines that we flagged as invalid. - Check if you are using the correct file format or if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) + module.Warning(zeroIPNotificationID, "Custom filter list has many invalid entries", + fmt.Sprintf(`%d out of %d entires are invalid. + Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) } else { module.Resolve(zeroIPNotificationID) } - log.Infof("intel/customlists: list loaded successful: %s", filePath) + allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList) + log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath) notifications.NotifyInfo(parseStatusNotificationID, "Custom filter list loaded successfully.", - fmt.Sprintf(`Custom filter list loaded successfully from file %s -%d domains + fmt.Sprintf(`Custom filter list loaded successfully from file %s - loaded: +%d Domains %d IPs -%d autonomous systems -%d countries`, +%d Autonomous Systems +%d Countries`, filePath, len(domainsFilterList), len(ipAddressesFilterList), len(autonomousSystemsFilterList), - len(domainsFilterList))) + len(countryCodesFilterList))) - module.Resolve(parseStatusNotificationID) + module.Resolve(parseWarningNotificationID) return nil } @@ -140,7 +141,7 @@ func parseLine(line string) bool { ip := net.ParseIP(field) if ip != nil { // check for zero ip. - if bytes.Compare(ip, net.IPv4zero) == 0 || bytes.Compare(ip, net.IPv6zero) == 0 { + if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { return false } diff --git a/intel/customlists/module.go b/intel/customlists/module.go index f0cdc721..fd918c60 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/safing/portbase/api" + "github.com/safing/portbase/api" //nolint // completing about import order!? "github.com/safing/portbase/modules" "golang.org/x/net/publicsuffix" ) @@ -58,7 +58,7 @@ func start() error { configChangeEvent, "update custom filter list", func(ctx context.Context, obj interface{}) error { - _ = checkAndUpdateFilterList() + checkAndUpdateFilterList() return nil }, ); err != nil { @@ -67,24 +67,21 @@ func start() error { // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { - _ = checkAndUpdateFilterList() + checkAndUpdateFilterList() return nil }).Schedule(time.Now().Add(20 * time.Second)) // register api endpoint for updating the filter list if err := api.RegisterEndpoint(api.Endpoint{ Path: "customlists/update", - Read: api.PermitUser, + Write: api.PermitUser, BelongsTo: module, ActionFunc: func(ar *api.Request) (msg string, err error) { - err = checkAndUpdateFilterList() - if err != nil { - return "failed to load custom filter list.", err - } - return "custom filter list loaded successfully.", nil + checkAndUpdateFilterList() + return "Custom filter list loaded successfully.", nil }, Name: "Update custom filter list", - Description: "Load a filter list form a file defined by the user.", + Description: "Reload the filter list from the configured file.", }); err != nil { return err } @@ -92,14 +89,14 @@ func start() error { return nil } -func checkAndUpdateFilterList() error { +func checkAndUpdateFilterList() { filterListLock.Lock() defer filterListLock.Unlock() // get path and ignore if empty filePath := getFilePath() if filePath == "" { - return nil + return } // schedule next update check @@ -115,12 +112,11 @@ func checkAndUpdateFilterList() error { if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { err := parseFile(filePath) if err != nil { - return nil + return } filterListFileModifiedTime = modifiedTime filterListFilePath = filePath } - return nil } // LookupIP checks if the IP address is in a custom filter list.