From 420da81b33b0367be42407c3b025931bedf40acd Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Fri, 22 Jul 2022 17:49:06 +0200 Subject: [PATCH 01/15] Custom filter list proof of concept --- firewall/master.go | 25 ++++++++ intel/customlists/module.go | 112 ++++++++++++++++++++++++++++++++++++ intel/module.go | 3 +- 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 intel/customlists/module.go diff --git a/firewall/master.go b/firewall/master.go index 1de7f194..0dbe21e9 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -13,6 +13,7 @@ import ( "github.com/safing/portbase/log" "github.com/safing/portmaster/detection/dga" + "github.com/safing/portmaster/intel/customlists" "github.com/safing/portmaster/netenv" "github.com/safing/portmaster/network" "github.com/safing/portmaster/network/netutils" @@ -54,6 +55,7 @@ var defaultDeciders = []deciderFn{ dropInbound, checkDomainHeuristics, checkAutoPermitRelated, + checkCustomFilterList, } // DecideOnConnection makes a decision about a connection. @@ -612,3 +614,26 @@ matchLoop: } return related, reason } + +func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { + // block domains that are in the custom list + if conn.Entity.Domain != "" { + if customlists.LookupDomain(conn.Entity.Domain) { + // FIXME: add proper messages + log.Debugf("Blocked %s", conn.Entity.Domain) + conn.Block("Domains appiers in the custom user list", profile.CfgOptionRemoveBlockedDNSKey) + return true + } + } + + if conn.Entity.IP != nil { + if customlists.LookupIPv4(&conn.Entity.IP) { + // FIXME: add proper messages + log.Debugf("Blocked %s", conn.Entity.IP) + conn.Block("IP appiers in the custom user list", profile.CfgOptionBlockScopeInternetKey) + return true + } + } + + return false +} diff --git a/intel/customlists/module.go b/intel/customlists/module.go new file mode 100644 index 00000000..129c01c8 --- /dev/null +++ b/intel/customlists/module.go @@ -0,0 +1,112 @@ +package customlists + +import ( + "bufio" + "net" + "os" + "regexp" + "strings" + + "github.com/miekg/dns" + "github.com/safing/portbase/log" + "github.com/safing/portbase/modules" + "github.com/safing/portmaster/network/netutils" +) + +var module *modules.Module + +// Helper variables for parsing the input file +var ( + countryCodes = map[string]struct{}{"AF": {}, "AX": {}, "AL": {}, "DZ": {}, "AS": {}, "AD": {}, "AO": {}, "AI": {}, "AQ": {}, "AG": {}, "AR": {}, "AM": {}, "AW": {}, "AU": {}, "AT": {}, "AZ": {}, "BH": {}, "BS": {}, "BD": {}, "BB": {}, "BY": {}, "BE": {}, "BZ": {}, "BJ": {}, "BM": {}, "BT": {}, "BO": {}, "BQ": {}, "BA": {}, "BW": {}, "BV": {}, "BR": {}, "IO": {}, "BN": {}, "BG": {}, "BF": {}, "BI": {}, "KH": {}, "CM": {}, "CA": {}, "CV": {}, "KY": {}, "CF": {}, "TD": {}, "CL": {}, "CN": {}, "CX": {}, "CC": {}, "CO": {}, "KM": {}, "CG": {}, "CD": {}, "CK": {}, "CR": {}, "CI": {}, "HR": {}, "CU": {}, "CW": {}, "CY": {}, "CZ": {}, "DK": {}, "DJ": {}, "DM": {}, "DO": {}, "EC": {}, "EG": {}, "SV": {}, "GQ": {}, "ER": {}, "EE": {}, "ET": {}, "FK": {}, "FO": {}, "FJ": {}, "FI": {}, "FR": {}, "GF": {}, "PF": {}, "TF": {}, "GA": {}, "GM": {}, "GE": {}, "DE": {}, "GH": {}, "GI": {}, "GR": {}, "GL": {}, "GD": {}, "GP": {}, "GU": {}, "GT": {}, "GG": {}, "GN": {}, "GW": {}, "GY": {}, "HT": {}, "HM": {}, "VA": {}, "HN": {}, "HK": {}, "HU": {}, "IS": {}, "IN": {}, "ID": {}, "IR": {}, "IQ": {}, "IE": {}, "IM": {}, "IL": {}, "IT": {}, "JM": {}, "JP": {}, "JE": {}, "JO": {}, "KZ": {}, "KE": {}, "KI": {}, "KP": {}, "KR": {}, "KW": {}, "KG": {}, "LA": {}, "LV": {}, "LB": {}, "LS": {}, "LR": {}, "LY": {}, "LI": {}, "LT": {}, "LU": {}, "MO": {}, "MK": {}, "MG": {}, "MW": {}, "MY": {}, "MV": {}, "ML": {}, "MT": {}, "MH": {}, "MQ": {}, "MR": {}, "MU": {}, "YT": {}, "MX": {}, "FM": {}, "MD": {}, "MC": {}, "MN": {}, "ME": {}, "MS": {}, "MA": {}, "MZ": {}, "MM": {}, "NA": {}, "NR": {}, "NP": {}, "NL": {}, "NC": {}, "NZ": {}, "NI": {}, "NE": {}, "NG": {}, "NU": {}, "NF": {}, "MP": {}, "NO": {}, "OM": {}, "PK": {}, "PW": {}, "PS": {}, "PA": {}, "PG": {}, "PY": {}, "PE": {}, "PH": {}, "PN": {}, "PL": {}, "PT": {}, "PR": {}, "QA": {}, "RE": {}, "RO": {}, "RU": {}, "RW": {}, "BL": {}, "SH": {}, "KN": {}, "LC": {}, "MF": {}, "PM": {}, "VC": {}, "WS": {}, "SM": {}, "ST": {}, "SA": {}, "SN": {}, "RS": {}, "SC": {}, "SL": {}, "SG": {}, "SX": {}, "SK": {}, "SI": {}, "SB": {}, "SO": {}, "ZA": {}, "GS": {}, "SS": {}, "ES": {}, "LK": {}, "SD": {}, "SR": {}, "SJ": {}, "SZ": {}, "SE": {}, "CH": {}, "SY": {}, "TW": {}, "TJ": {}, "TZ": {}, "TH": {}, "TL": {}, "TG": {}, "TK": {}, "TO": {}, "TT": {}, "TN": {}, "TR": {}, "TM": {}, "TC": {}, "TV": {}, "UG": {}, "UA": {}, "AE": {}, "GB": {}, "US": {}, "UM": {}, "UY": {}, "UZ": {}, "VU": {}, "VE": {}, "VN": {}, "VG": {}, "VI": {}, "WF": {}, "EH": {}, "YE": {}, "ZM": {}, "ZW": {}} + isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString +) + +var ( + filteredCountryCodes map[string]struct{} + filteredIPAddresses map[string]struct{} + filteredAutonomousSystems map[string]struct{} + filteredDomains map[string]struct{} +) + +func init() { + module = modules.Register("customlists", prep, nil, nil, "base") +} + +func prep() error { + filteredCountryCodes = make(map[string]struct{}) + filteredIPAddresses = make(map[string]struct{}) + filteredAutonomousSystems = make(map[string]struct{}) + filteredDomains = make(map[string]struct{}) + + file, err := os.Open("/home/vladimir/Dev/Safing/filterlists/custom.txt") + if err != nil { + return err + } + defer file.Close() + + // read filter file line by line + scanner := bufio.NewScanner(file) + // the scanner will error out if the line is greater than 64K, in this case it is enough + for scanner.Scan() { + parseFilterLine(scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return err + } + + log.Criticalf("filteredCountryCodes: %v", filteredCountryCodes) + log.Criticalf("filteredIPAddresses: %v", filteredIPAddresses) + log.Criticalf("filteredAutonomousSystems: %v", filteredAutonomousSystems) + log.Criticalf("filteredDomains: %v", filteredDomains) + + return nil +} + +func parseFilterLine(line string) { + // ignore empty lines and comment lines + if len(line) == 0 || line[0] == '#' { + return + } + + fields := strings.Fields(line) + + // everything after the first field will be ignored + firstField := fields[0] + + // check if it'a a country code + if _, ok := countryCodes[firstField]; ok { + filteredCountryCodes[firstField] = struct{}{} + } + + // try to parse IP address + ip := net.ParseIP(firstField) + if ip != nil { + filteredIPAddresses[ip.String()] = struct{}{} + } + + // check if it's a Autonomous system (example AS123) + if isAutonomousSystem(firstField) { + filteredAutonomousSystems[firstField] = struct{}{} + } + + // check if it's a domain + potentialDomain := dns.Fqdn(firstField) + if netutils.IsValidFqdn(potentialDomain) { + filteredDomains[potentialDomain] = struct{}{} + } +} + +// LookupIPv4 checks if the IP is in a custom filter list +func LookupIPv4(ip *net.IP) bool { + log.Debugf("Checking ip %s", ip.String()) + _, ok := filteredIPAddresses[ip.String()] + return ok +} + +// LookupDomain checks if the Domain is in a custom filter list +func LookupDomain(domain string) bool { + log.Debugf("Checking domain %s", domain) + _, ok := filteredDomains[domain] + return ok +} diff --git a/intel/module.go b/intel/module.go index 65fcc65d..ceec6b64 100644 --- a/intel/module.go +++ b/intel/module.go @@ -2,11 +2,12 @@ package intel import ( "github.com/safing/portbase/modules" + _ "github.com/safing/portmaster/intel/customlists" ) // Module of this package. Export needed for testing of the endpoints package. var Module *modules.Module func init() { - Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists") + Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists", "customlists") } From 35697989e598870d25c2ea57ef84dc7ee3f8c56a Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Tue, 26 Jul 2022 07:10:03 +0200 Subject: [PATCH 02/15] Refactoring --- firewall/master.go | 29 +++++--- intel/customlists/config.go | 39 +++++++++++ intel/customlists/lists.go | 88 +++++++++++++++++++++++ intel/customlists/module.go | 134 ++++++++++++++++++------------------ 4 files changed, 214 insertions(+), 76 deletions(-) create mode 100644 intel/customlists/config.go create mode 100644 intel/customlists/lists.go diff --git a/firewall/master.go b/firewall/master.go index 0dbe21e9..fee5adba 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -616,21 +616,34 @@ matchLoop: } func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { - // block domains that are in the custom list + // block if the domain name appears in the custom filter list if conn.Entity.Domain != "" { if customlists.LookupDomain(conn.Entity.Domain) { - // FIXME: add proper messages - log.Debugf("Blocked %s", conn.Entity.Domain) - conn.Block("Domains appiers in the custom user list", profile.CfgOptionRemoveBlockedDNSKey) + conn.Block("Domains appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) return true } } + // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { - if customlists.LookupIPv4(&conn.Entity.IP) { - // FIXME: add proper messages - log.Debugf("Blocked %s", conn.Entity.IP) - conn.Block("IP appiers in the custom user list", profile.CfgOptionBlockScopeInternetKey) + if customlists.LookupIP(&conn.Entity.IP) { + conn.Block("IP appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + // block autonomous system by its number if it appears in the custom filter list + if conn.Entity.ASN != 0 { + if customlists.LookupASN(conn.Entity.ASN) { + conn.Block("ASN appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + // block if the country appears in the custom filter list + if conn.Entity.Country != "" { + if customlists.LookupCountry(conn.Entity.Country) { + conn.Block("Country appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } diff --git a/intel/customlists/config.go b/intel/customlists/config.go new file mode 100644 index 00000000..f011827a --- /dev/null +++ b/intel/customlists/config.go @@ -0,0 +1,39 @@ +package customlists + +import "github.com/safing/portbase/config" + +var ( + // CfgOptionCustomListBlockingKey is the config key for the listen address.. + CfgOptionCustomListBlockingKey = "filter/customListBlocking" + cfgOptionCustomListBlockingOrder = 37 + cfgOptionCustomListCategoryAnnotation = "Filter Lists" +) + +var ( + getFilePath func() string +) + +func registerConfig() error { + // register a setting for the file path in the ui + err := config.Register(&config.Option{ + Name: "Custom Filter List", + Key: CfgOptionCustomListBlockingKey, + Description: "Path to the file that contains a list of Domain, IP addresses, country codes and autonomous systems that you want to block", + OptType: config.OptTypeString, + ExpertiseLevel: config.ExpertiseLevelExpert, + ReleaseLevel: config.ReleaseLevelStable, + DefaultValue: "", + RequiresRestart: false, + Annotations: config.Annotations{ + config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder, + config.CategoryAnnotation: cfgOptionCustomListCategoryAnnotation, + }, + }) + if err != nil { + return err + } + + getFilePath = config.GetAsString(CfgOptionCustomListBlockingKey, "") + + return nil +} diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go new file mode 100644 index 00000000..c9d0f410 --- /dev/null +++ b/intel/customlists/lists.go @@ -0,0 +1,88 @@ +package customlists + +import ( + "bufio" + "net" + "os" + "strconv" + "strings" + + "github.com/miekg/dns" + "github.com/safing/portbase/log" + "github.com/safing/portmaster/network/netutils" +) + +var ( + countryCodesFilterList map[string]struct{} + ipAddressesFilterList map[string]struct{} + autonomousSystemsFilterList map[uint]struct{} + domainsFilterList map[string]struct{} +) + +func parseFile(filePath string) error { + // open the file if possible + file, err := os.Open(filePath) + if err != nil { + log.Warningf("Custom filter: failed to parse file: \"%s\"", filePath) + return err + } + defer file.Close() + + // initialize maps to hold data from the file + countryCodesFilterList = make(map[string]struct{}) + ipAddressesFilterList = make(map[string]struct{}) + autonomousSystemsFilterList = make(map[uint]struct{}) + domainsFilterList = make(map[string]struct{}) + + // read filter file line by line + scanner := bufio.NewScanner(file) + // the scanner will error out if the line is greater than 64K, in this case it is enough + for scanner.Scan() { + parseLine(scanner.Text()) + } + + // check for scanner error + if err := scanner.Err(); err != nil { + return err + } + + log.Infof("Custom filter: list loaded successful: %s", filePath) + + return nil +} + +func parseLine(line string) { + // ignore empty lines and comment lines + if len(line) == 0 || line[0] == '#' { + return + } + + // everything after the first field will be ignored + field := strings.Fields(line)[0] + + // check if it'a a country code + if isCountryCode(field) { + countryCodesFilterList[field] = struct{}{} + } + + // try to parse IP address + ip := net.ParseIP(field) + if ip != nil { + ipAddressesFilterList[ip.String()] = struct{}{} + } + + // check if it's a Autonomous system (example AS123) + if isAutonomousSystem(field) { + asNumber, err := strconv.ParseUint(field[2:], 10, 32) + if err != nil { + return + } + autonomousSystemsFilterList[uint(asNumber)] = struct{}{} + } + + // check if it's a domain + domain := dns.Fqdn(field) + if netutils.IsValidFqdn(domain) { + domainsFilterList[domain] = struct{}{} + } +} diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 129c01c8..2d5feba6 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -1,112 +1,110 @@ package customlists import ( - "bufio" + "context" "net" "os" "regexp" - "strings" + "time" - "github.com/miekg/dns" - "github.com/safing/portbase/log" "github.com/safing/portbase/modules" - "github.com/safing/portmaster/network/netutils" ) var module *modules.Module +const configChangeEvent = "config change" + // Helper variables for parsing the input file var ( - countryCodes = map[string]struct{}{"AF": {}, "AX": {}, "AL": {}, "DZ": {}, "AS": {}, "AD": {}, "AO": {}, "AI": {}, "AQ": {}, "AG": {}, "AR": {}, "AM": {}, "AW": {}, "AU": {}, "AT": {}, "AZ": {}, "BH": {}, "BS": {}, "BD": {}, "BB": {}, "BY": {}, "BE": {}, "BZ": {}, "BJ": {}, "BM": {}, "BT": {}, "BO": {}, "BQ": {}, "BA": {}, "BW": {}, "BV": {}, "BR": {}, "IO": {}, "BN": {}, "BG": {}, "BF": {}, "BI": {}, "KH": {}, "CM": {}, "CA": {}, "CV": {}, "KY": {}, "CF": {}, "TD": {}, "CL": {}, "CN": {}, "CX": {}, "CC": {}, "CO": {}, "KM": {}, "CG": {}, "CD": {}, "CK": {}, "CR": {}, "CI": {}, "HR": {}, "CU": {}, "CW": {}, "CY": {}, "CZ": {}, "DK": {}, "DJ": {}, "DM": {}, "DO": {}, "EC": {}, "EG": {}, "SV": {}, "GQ": {}, "ER": {}, "EE": {}, "ET": {}, "FK": {}, "FO": {}, "FJ": {}, "FI": {}, "FR": {}, "GF": {}, "PF": {}, "TF": {}, "GA": {}, "GM": {}, "GE": {}, "DE": {}, "GH": {}, "GI": {}, "GR": {}, "GL": {}, "GD": {}, "GP": {}, "GU": {}, "GT": {}, "GG": {}, "GN": {}, "GW": {}, "GY": {}, "HT": {}, "HM": {}, "VA": {}, "HN": {}, "HK": {}, "HU": {}, "IS": {}, "IN": {}, "ID": {}, "IR": {}, "IQ": {}, "IE": {}, "IM": {}, "IL": {}, "IT": {}, "JM": {}, "JP": {}, "JE": {}, "JO": {}, "KZ": {}, "KE": {}, "KI": {}, "KP": {}, "KR": {}, "KW": {}, "KG": {}, "LA": {}, "LV": {}, "LB": {}, "LS": {}, "LR": {}, "LY": {}, "LI": {}, "LT": {}, "LU": {}, "MO": {}, "MK": {}, "MG": {}, "MW": {}, "MY": {}, "MV": {}, "ML": {}, "MT": {}, "MH": {}, "MQ": {}, "MR": {}, "MU": {}, "YT": {}, "MX": {}, "FM": {}, "MD": {}, "MC": {}, "MN": {}, "ME": {}, "MS": {}, "MA": {}, "MZ": {}, "MM": {}, "NA": {}, "NR": {}, "NP": {}, "NL": {}, "NC": {}, "NZ": {}, "NI": {}, "NE": {}, "NG": {}, "NU": {}, "NF": {}, "MP": {}, "NO": {}, "OM": {}, "PK": {}, "PW": {}, "PS": {}, "PA": {}, "PG": {}, "PY": {}, "PE": {}, "PH": {}, "PN": {}, "PL": {}, "PT": {}, "PR": {}, "QA": {}, "RE": {}, "RO": {}, "RU": {}, "RW": {}, "BL": {}, "SH": {}, "KN": {}, "LC": {}, "MF": {}, "PM": {}, "VC": {}, "WS": {}, "SM": {}, "ST": {}, "SA": {}, "SN": {}, "RS": {}, "SC": {}, "SL": {}, "SG": {}, "SX": {}, "SK": {}, "SI": {}, "SB": {}, "SO": {}, "ZA": {}, "GS": {}, "SS": {}, "ES": {}, "LK": {}, "SD": {}, "SR": {}, "SJ": {}, "SZ": {}, "SE": {}, "CH": {}, "SY": {}, "TW": {}, "TJ": {}, "TZ": {}, "TH": {}, "TL": {}, "TG": {}, "TK": {}, "TO": {}, "TT": {}, "TN": {}, "TR": {}, "TM": {}, "TC": {}, "TV": {}, "UG": {}, "UA": {}, "AE": {}, "GB": {}, "US": {}, "UM": {}, "UY": {}, "UZ": {}, "VU": {}, "VE": {}, "VN": {}, "VG": {}, "VI": {}, "WF": {}, "EH": {}, "YE": {}, "ZM": {}, "ZW": {}} + isCountryCode = regexp.MustCompile("^[A-Z]{2}$").MatchString isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString ) var ( - filteredCountryCodes map[string]struct{} - filteredIPAddresses map[string]struct{} - filteredAutonomousSystems map[string]struct{} - filteredDomains map[string]struct{} + filterListFilePath string + filterListFileModifiedTime time.Time ) func init() { - module = modules.Register("customlists", prep, nil, nil, "base") + module = modules.Register("customlists", prep, start, nil, "base") } func prep() error { - filteredCountryCodes = make(map[string]struct{}) - filteredIPAddresses = make(map[string]struct{}) - filteredAutonomousSystems = make(map[string]struct{}) - filteredDomains = make(map[string]struct{}) - - file, err := os.Open("/home/vladimir/Dev/Safing/filterlists/custom.txt") + // register the config in the ui + err := registerConfig() if err != nil { return err } - defer file.Close() - // read filter file line by line - scanner := bufio.NewScanner(file) - // the scanner will error out if the line is greater than 64K, in this case it is enough - for scanner.Scan() { - parseFilterLine(scanner.Text()) - } - - if err := scanner.Err(); err != nil { + // register to hook to update after config change. + if err := module.RegisterEventHook( + module.Name, + configChangeEvent, + "update custom filter list", + func(ctx context.Context, obj interface{}) error { + _ = checkAndUpdateFilterList() + return nil + }, + ); err != nil { return err } - log.Criticalf("filteredCountryCodes: %v", filteredCountryCodes) - log.Criticalf("filteredIPAddresses: %v", filteredIPAddresses) - log.Criticalf("filteredAutonomousSystems: %v", filteredAutonomousSystems) - log.Criticalf("filteredDomains: %v", filteredDomains) - return nil } -func parseFilterLine(line string) { - // ignore empty lines and comment lines - if len(line) == 0 || line[0] == '#' { - return - } +func start() error { + // register timer to run every periodically and check for file updates + module.NewTask("Custom filter list file update check", func(context.Context, *modules.Task) error { + _ = checkAndUpdateFilterList() + return nil + }).Repeat(10 * time.Minute) - fields := strings.Fields(line) - - // everything after the first field will be ignored - firstField := fields[0] - - // check if it'a a country code - if _, ok := countryCodes[firstField]; ok { - filteredCountryCodes[firstField] = struct{}{} - } - - // try to parse IP address - ip := net.ParseIP(firstField) - if ip != nil { - filteredIPAddresses[ip.String()] = struct{}{} - } - - // check if it's a Autonomous system (example AS123) - if isAutonomousSystem(firstField) { - filteredAutonomousSystems[firstField] = struct{}{} - } - - // check if it's a domain - potentialDomain := dns.Fqdn(firstField) - if netutils.IsValidFqdn(potentialDomain) { - filteredDomains[potentialDomain] = struct{}{} - } + // parse the file for the first time at start + _ = parseFile(getFilePath()) + return nil } -// LookupIPv4 checks if the IP is in a custom filter list -func LookupIPv4(ip *net.IP) bool { - log.Debugf("Checking ip %s", ip.String()) - _, ok := filteredIPAddresses[ip.String()] +func checkAndUpdateFilterList() error { + // get path and try to get its info + filePath := getFilePath() + fileInfo, err := os.Stat(filePath) + if err != nil { + return nil + } + modifiedTime := fileInfo.ModTime() + + // check if file path has changed or if modified time has changed + if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { + err := parseFile(filePath) + if err != nil { + return nil + } + filterListFileModifiedTime = modifiedTime + filterListFilePath = filePath + } + return nil +} + +// LookupIP checks if the IP address is in a custom filter list +func LookupIP(ip *net.IP) bool { + _, ok := ipAddressesFilterList[ip.String()] return ok } // LookupDomain checks if the Domain is in a custom filter list func LookupDomain(domain string) bool { - log.Debugf("Checking domain %s", domain) - _, ok := filteredDomains[domain] + _, ok := domainsFilterList[domain] + return ok +} + +// LookupASN checks if the Autonomous system number is in a custom filter list +func LookupASN(number uint) bool { + _, ok := autonomousSystemsFilterList[number] + return ok +} + +// LookupCountry checks if the country code is in a custom filter list +func LookupCountry(countryCode string) bool { + _, ok := countryCodesFilterList[countryCode] return ok } From 62c100714a68053680911483e08501e07d156a2d Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Tue, 26 Jul 2022 17:18:58 +0200 Subject: [PATCH 03/15] Custom filter list: subdomain and cname cheks Automatic realod when settings is changed periodicly check for file changes --- firewall/master.go | 14 +++++++-- intel/customlists/config.go | 32 +++++++++++++++++-- intel/customlists/lists.go | 30 +++++++++++++++--- intel/customlists/module.go | 62 ++++++++++++++++++++++++++++++++----- profile/config.go | 4 ++- 5 files changed, 126 insertions(+), 16 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index fee5adba..ab368cf3 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -616,14 +616,24 @@ matchLoop: } func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { - // block if the domain name appears in the custom filter list + // block if the domain name appears in the custom filter list (check for subdomains if enabled) if conn.Entity.Domain != "" { - if customlists.LookupDomain(conn.Entity.Domain) { + if customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()) { conn.Block("Domains appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) return true } } + // block if any of the CNAME appears in the custom filter list (check for subdomains if enabled) + if len(conn.Entity.CNAME) > 0 && p.FilterCNAMEs() { + for _, cname := range conn.Entity.CNAME { + if customlists.LookupDomain(cname, p.FilterSubDomains()) { + conn.Block("CNAME appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + } + // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { if customlists.LookupIP(&conn.Entity.IP) { diff --git a/intel/customlists/config.go b/intel/customlists/config.go index f011827a..08a947b6 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -1,11 +1,15 @@ package customlists -import "github.com/safing/portbase/config" +import ( + "strings" + + "github.com/safing/portbase/config" +) var ( // CfgOptionCustomListBlockingKey is the config key for the listen address.. CfgOptionCustomListBlockingKey = "filter/customListBlocking" - cfgOptionCustomListBlockingOrder = 37 + cfgOptionCustomListBlockingOrder = 35 cfgOptionCustomListCategoryAnnotation = "Filter Lists" ) @@ -14,11 +18,35 @@ var ( ) func registerConfig() error { + help := strings.ReplaceAll(`Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in which each entry is on a new line. + Lines that start with a '#' symbol are ignored. + Everything after the first space/tab is ignored. + Example: +""" +# Domains: +example.com +google.com + +# IP addresses +1.2.3.4 +4.3.2.1 + +# Countries +AU +BG + +# Autonomous Systems +AS123 +""" +> * All the records are stored in RAM, careful with large block lists. +> * Hosts files are not supported.`, "\"", "`") + // register a setting for the file path in the ui err := config.Register(&config.Option{ Name: "Custom Filter List", Key: CfgOptionCustomListBlockingKey, Description: "Path to the file that contains a list of Domain, IP addresses, country codes and autonomous systems that you want to block", + Help: help, OptType: config.OptTypeString, ExpertiseLevel: config.ExpertiseLevelExpert, ReleaseLevel: config.ReleaseLevelStable, diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index c9d0f410..753541fe 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -9,6 +9,7 @@ import ( "github.com/miekg/dns" "github.com/safing/portbase/log" + "github.com/safing/portbase/notifications" "github.com/safing/portmaster/network/netutils" ) @@ -19,11 +20,18 @@ var ( domainsFilterList map[string]struct{} ) +const numberOfZeroIPsUntilWarning = 100 + func parseFile(filePath string) error { + // ignore empty file path + if filePath == "" { + return nil + } + // open the file if possible file, err := os.Open(filePath) if err != nil { - log.Warningf("Custom filter: failed to parse file: \"%s\"", filePath) + log.Warningf("intel/customlists: failed to parse file: \"%s\"", filePath) return err } defer file.Close() @@ -34,11 +42,13 @@ func parseFile(filePath string) error { autonomousSystemsFilterList = make(map[uint]struct{}) domainsFilterList = make(map[string]struct{}) + var numberOfZeroIPs uint64 + // read filter file line by line scanner := bufio.NewScanner(file) // the scanner will error out if the line is greater than 64K, in this case it is enough for scanner.Scan() { - parseLine(scanner.Text()) + parseLine(scanner.Text(), &numberOfZeroIPs) } // check for scanner error @@ -46,12 +56,17 @@ func parseFile(filePath string) error { return err } - log.Infof("Custom filter: list loaded successful: %s", filePath) + if numberOfZeroIPs >= numberOfZeroIPsUntilWarning { + log.Warning("intel/customlists: Too many zero IP addresses.") + notifications.NotifyWarn("too_many_zero_ips", "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + } + + log.Infof("intel/customlists: list loaded successful: %s", filePath) return nil } -func parseLine(line string) { +func parseLine(line string, numberOfZeroIPs *uint64) { // ignore empty lines and comment lines if len(line) == 0 || line[0] == '#' { return @@ -69,6 +84,13 @@ func parseLine(line string) { ip := net.ParseIP(field) if ip != nil { ipAddressesFilterList[ip.String()] = struct{}{} + + // check if its zero ip + for i := 0; i < len(ip); i++ { + if ip[i] != 0 { + *numberOfZeroIPs++ + } + } } // check if it's a Autonomous system (example AS123) diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 2d5feba6..18c90c8f 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -5,14 +5,20 @@ import ( "net" "os" "regexp" + "strings" + "sync" "time" "github.com/safing/portbase/modules" + "golang.org/x/net/publicsuffix" ) var module *modules.Module -const configChangeEvent = "config change" +const ( + configModuleName = "config" + configChangeEvent = "config change" +) // Helper variables for parsing the input file var ( @@ -23,6 +29,8 @@ var ( var ( filterListFilePath string filterListFileModifiedTime time.Time + + parseLock sync.RWMutex ) func init() { @@ -38,7 +46,7 @@ func prep() error { // register to hook to update after config change. if err := module.RegisterEventHook( - module.Name, + configModuleName, configChangeEvent, "update custom filter list", func(ctx context.Context, obj interface{}) error { @@ -54,17 +62,20 @@ func prep() error { func start() error { // register timer to run every periodically and check for file updates - module.NewTask("Custom filter list file update check", func(context.Context, *modules.Task) error { + module.NewTask("intel/customlists list file update check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() return nil }).Repeat(10 * time.Minute) - // parse the file for the first time at start + // parse the file at startup _ = parseFile(getFilePath()) return nil } func checkAndUpdateFilterList() error { + parseLock.Lock() + defer parseLock.Unlock() + // get path and try to get its info filePath := getFilePath() fileInfo, err := os.Stat(filePath) @@ -92,9 +103,20 @@ func LookupIP(ip *net.IP) bool { } // LookupDomain checks if the Domain is in a custom filter list -func LookupDomain(domain string) bool { - _, ok := domainsFilterList[domain] - return ok +func LookupDomain(fullDomain string, filterSubdomains bool) bool { + if filterSubdomains { + listOfDomains := splitDomain(fullDomain) + for _, domain := range listOfDomains { + _, ok := domainsFilterList[domain] + if ok { + return true + } + } + } else { + _, ok := domainsFilterList[fullDomain] + return ok + } + return false } // LookupASN checks if the Autonomous system number is in a custom filter list @@ -108,3 +130,29 @@ func LookupCountry(countryCode string) bool { _, ok := countryCodesFilterList[countryCode] return ok } + +func splitDomain(domain string) []string { + domain = strings.Trim(domain, ".") + suffix, _ := publicsuffix.PublicSuffix(domain) + if suffix == domain { + return []string{domain} + } + + domainWithoutSuffix := domain[:len(domain)-len(suffix)] + domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".") + + splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool { + return r == '.' + }) + + domains := make([]string, 0, len(splitted)) + for idx := range splitted { + + d := strings.Join(splitted[idx:], ".") + "." + suffix + if d[len(d)-1] != '.' { + d += "." + } + domains = append(domains, d) + } + return domains +} diff --git a/profile/config.go b/profile/config.go index e371a175..3df5e3be 100644 --- a/profile/config.go +++ b/profile/config.go @@ -64,9 +64,11 @@ var ( cfgOptionFilterLists config.StringArrayOption cfgOptionFilterListsOrder = 34 + // Setting "Custom Filter List" at order 35 + CfgOptionFilterSubDomainsKey = "filter/includeSubdomains" cfgOptionFilterSubDomains config.IntOption // security level option - cfgOptionFilterSubDomainsOrder = 35 + cfgOptionFilterSubDomainsOrder = 36 // DNS Filtering. From cf30014e9da8ac9a6d75966cb4cfa9e7acab2d88 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Wed, 27 Jul 2022 07:47:48 +0200 Subject: [PATCH 04/15] Bug fixes for custom filter list --- intel/customlists/config.go | 24 +++++++++++------------- intel/customlists/lists.go | 12 ++++++------ intel/customlists/module.go | 11 ++++++----- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 08a947b6..9b8e0ba7 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -1,8 +1,6 @@ package customlists import ( - "strings" - "github.com/safing/portbase/config" ) @@ -18,28 +16,28 @@ var ( ) func registerConfig() error { - help := strings.ReplaceAll(`Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in which each entry is on a new line. - Lines that start with a '#' symbol are ignored. - Everything after the first space/tab is ignored. - Example: -""" -# Domains: + help := `Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in where each entry is on a new line. +Lines that start with a '#' symbol are ignored. +Everything after the first space/tab is ignored. +Example: +############# +\# Domains: example.com google.com -# IP addresses +\# IP addresses 1.2.3.4 4.3.2.1 -# Countries +\# Countries AU BG -# Autonomous Systems +\# Autonomous Systems AS123 -""" +############# > * All the records are stored in RAM, careful with large block lists. -> * Hosts files are not supported.`, "\"", "`") +> * Hosts files are not supported.` // register a setting for the file path in the ui err := config.Register(&config.Option{ diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 753541fe..ad066d0d 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -23,6 +23,12 @@ var ( const numberOfZeroIPsUntilWarning = 100 func parseFile(filePath string) error { + // reset all maps, previous (if any) settings will be lost + countryCodesFilterList = make(map[string]struct{}) + ipAddressesFilterList = make(map[string]struct{}) + autonomousSystemsFilterList = make(map[uint]struct{}) + domainsFilterList = make(map[string]struct{}) + // ignore empty file path if filePath == "" { return nil @@ -36,12 +42,6 @@ func parseFile(filePath string) error { } defer file.Close() - // initialize maps to hold data from the file - countryCodesFilterList = make(map[string]struct{}) - ipAddressesFilterList = make(map[string]struct{}) - autonomousSystemsFilterList = make(map[uint]struct{}) - domainsFilterList = make(map[string]struct{}) - var numberOfZeroIPs uint64 // read filter file line by line diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 18c90c8f..03d87f67 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -62,7 +62,7 @@ func prep() error { func start() error { // register timer to run every periodically and check for file updates - module.NewTask("intel/customlists list file update check", func(context.Context, *modules.Task) error { + module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() return nil }).Repeat(10 * time.Minute) @@ -78,11 +78,10 @@ func checkAndUpdateFilterList() error { // get path and try to get its info filePath := getFilePath() - fileInfo, err := os.Stat(filePath) - if err != nil { - return nil + modifiedTime := time.Now() + if fileInfo, err := os.Stat(filePath); err == nil { + modifiedTime = fileInfo.ModTime() } - modifiedTime := fileInfo.ModTime() // check if file path has changed or if modified time has changed if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { @@ -105,6 +104,7 @@ func LookupIP(ip *net.IP) bool { // LookupDomain checks if the Domain is in a custom filter list func LookupDomain(fullDomain string, filterSubdomains bool) bool { if filterSubdomains { + // check if domain is in the list and all its subdomains listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] @@ -113,6 +113,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) bool { } } } else { + //check only if the domain is in the list _, ok := domainsFilterList[fullDomain] return ok } From 1e79fedf9b07e0f51cc5dc4520e5f9830827e53d Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Wed, 27 Jul 2022 14:14:32 +0200 Subject: [PATCH 05/15] muve register event hook frop prep to start --- intel/customlists/module.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 03d87f67..e8ceb500 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -44,6 +44,10 @@ func prep() error { return err } + return nil +} + +func start() error { // register to hook to update after config change. if err := module.RegisterEventHook( configModuleName, @@ -57,10 +61,6 @@ func prep() error { return err } - return nil -} - -func start() error { // register timer to run every periodically and check for file updates module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() From 6a6468418b47916ba8e9ec113e1e126c70f385ec Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Wed, 27 Jul 2022 14:27:55 +0200 Subject: [PATCH 06/15] Fix linter errors --- intel/customlists/module.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/intel/customlists/module.go b/intel/customlists/module.go index e8ceb500..15029358 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -20,7 +20,7 @@ const ( configChangeEvent = "config change" ) -// Helper variables for parsing the input file +// Helper variables for parsing the input file. var ( isCountryCode = regexp.MustCompile("^[A-Z]{2}$").MatchString isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString @@ -95,13 +95,13 @@ func checkAndUpdateFilterList() error { return nil } -// LookupIP checks if the IP address is in a custom filter list +// LookupIP checks if the IP address is in a custom filter list. func LookupIP(ip *net.IP) bool { _, ok := ipAddressesFilterList[ip.String()] return ok } -// LookupDomain checks if the Domain is in a custom filter list +// LookupDomain checks if the Domain is in a custom filter list. func LookupDomain(fullDomain string, filterSubdomains bool) bool { if filterSubdomains { // check if domain is in the list and all its subdomains @@ -113,20 +113,20 @@ func LookupDomain(fullDomain string, filterSubdomains bool) bool { } } } else { - //check only if the domain is in the list + // check only if the domain is in the list _, ok := domainsFilterList[fullDomain] return ok } return false } -// LookupASN checks if the Autonomous system number is in a custom filter list +// LookupASN checks if the Autonomous system number is in a custom filter list. func LookupASN(number uint) bool { _, ok := autonomousSystemsFilterList[number] return ok } -// LookupCountry checks if the country code is in a custom filter list +// LookupCountry checks if the country code is in a custom filter list. func LookupCountry(countryCode string) bool { _, ok := countryCodesFilterList[countryCode] return ok From cb08bb293173c8a6dbfcf57c84839e3746f04832 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Wed, 27 Jul 2022 14:32:55 +0200 Subject: [PATCH 07/15] Fix more linter errors --- profile/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profile/config.go b/profile/config.go index 3df5e3be..8963d2ad 100644 --- a/profile/config.go +++ b/profile/config.go @@ -64,7 +64,7 @@ var ( cfgOptionFilterLists config.StringArrayOption cfgOptionFilterListsOrder = 34 - // Setting "Custom Filter List" at order 35 + // Setting "Custom Filter List" at order 35. CfgOptionFilterSubDomainsKey = "filter/includeSubdomains" cfgOptionFilterSubDomains config.IntOption // security level option From 93367b64df33ce58d74c3af63947b518c47ad88d Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Fri, 29 Jul 2022 16:48:53 +0200 Subject: [PATCH 08/15] Bug fixes and performence fixes: Synchronization bug fixed map reseting performence improvment Added more notificationa and better wording better scheduling of file parsing task --- firewall/master.go | 20 ++++++------- intel/customlists/config.go | 3 +- intel/customlists/lists.go | 57 +++++++++++++++++++++++++++++++++---- intel/customlists/module.go | 49 ++++++++++++++++++++++--------- 4 files changed, 99 insertions(+), 30 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index ab368cf3..50d64bc0 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -51,11 +51,11 @@ var defaultDeciders = []deciderFn{ checkResolverScope, checkConnectivityDomain, checkBypassPrevention, + checkCustomFilterList, checkFilterLists, dropInbound, checkDomainHeuristics, checkAutoPermitRelated, - checkCustomFilterList, } // DecideOnConnection makes a decision about a connection. @@ -618,17 +618,17 @@ matchLoop: func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { // block if the domain name appears in the custom filter list (check for subdomains if enabled) if conn.Entity.Domain != "" { - if customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()) { - conn.Block("Domains appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) + if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain %s matched %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) return true } } // block if any of the CNAME appears in the custom filter list (check for subdomains if enabled) - if len(conn.Entity.CNAME) > 0 && p.FilterCNAMEs() { + if p.FilterCNAMEs() { for _, cname := range conn.Entity.CNAME { - if customlists.LookupDomain(cname, p.FilterSubDomains()) { - conn.Block("CNAME appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) + if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matched %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -636,8 +636,8 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { - if customlists.LookupIP(&conn.Entity.IP) { - conn.Block("IP appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + if customlists.LookupIP(conn.Entity.IP) { + conn.Deny(fmt.Sprintf("IP address %s appears in the custom filter list", conn.Entity.IP), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -645,7 +645,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block autonomous system by its number if it appears in the custom filter list if conn.Entity.ASN != 0 { if customlists.LookupASN(conn.Entity.ASN) { - conn.Block("ASN appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("autonomous system with number %d appears in the custom filter list", conn.Entity.ASN), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -653,7 +653,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the country appears in the custom filter list if conn.Entity.Country != "" { if customlists.LookupCountry(conn.Entity.Country) { - conn.Block("Country appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("country code %s appears in the custom filter list", conn.Entity.Country), customlists.CfgOptionCustomListBlockingKey) return true } } diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 9b8e0ba7..33e285a9 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -16,7 +16,7 @@ var ( ) func registerConfig() error { - help := `Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in where each entry is on a new line. + help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. Lines that start with a '#' symbol are ignored. Everything after the first space/tab is ignored. Example: @@ -53,6 +53,7 @@ AS123 Annotations: config.Annotations{ config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder, config.CategoryAnnotation: cfgOptionCustomListCategoryAnnotation, + config.DisplayHintAnnotation: config.DisplayHintFilePicker, }, }) if err != nil { diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index ad066d0d..e9cbb90a 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -2,6 +2,7 @@ package customlists import ( "bufio" + "fmt" "net" "os" "strconv" @@ -20,14 +21,33 @@ var ( domainsFilterList map[string]struct{} ) -const numberOfZeroIPsUntilWarning = 100 +const ( + numberOfZeroIPsUntilWarning = 100 + customFilterListStatusNotificationID = "intel/customlists_status" + customFilterListZeroIPNotificationID = "intel/customlists_zeroip" +) -func parseFile(filePath string) error { - // reset all maps, previous (if any) settings will be lost +func initFilterLists() { countryCodesFilterList = make(map[string]struct{}) ipAddressesFilterList = make(map[string]struct{}) autonomousSystemsFilterList = make(map[uint]struct{}) domainsFilterList = make(map[string]struct{}) +} + +func parseFile(filePath string) error { + // reset all maps, previous (if any) settings will be lost + for key := range countryCodesFilterList { + delete(countryCodesFilterList, key) + } + for key := range ipAddressesFilterList { + delete(ipAddressesFilterList, key) + } + for key := range autonomousSystemsFilterList { + delete(autonomousSystemsFilterList, key) + } + for key := range domainsFilterList { + delete(domainsFilterList, key) + } // ignore empty file path if filePath == "" { @@ -37,7 +57,21 @@ func parseFile(filePath string) error { // open the file if possible file, err := os.Open(filePath) if err != nil { - log.Warningf("intel/customlists: failed to parse file: \"%s\"", filePath) + log.Warningf("intel/customlists: failed to parse file %q ", err) + // notifications.NotifyWarn("intel/customlists parse failed", "Failed to open custom filter list") + notifications.Notify(¬ifications.Notification{ + EventID: customFilterListStatusNotificationID, + Type: notifications.Warning, + Title: "Failed to open custom filter list", + Message: err.Error(), + ShowOnSystem: false, + AvailableActions: []*notifications.Action{ + { + ID: "ack", + Text: "OK", + }, + }, + }) return err } defer file.Close() @@ -58,11 +92,24 @@ func parseFile(filePath string) error { if numberOfZeroIPs >= numberOfZeroIPsUntilWarning { log.Warning("intel/customlists: Too many zero IP addresses.") - notifications.NotifyWarn("too_many_zero_ips", "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + notifications.NotifyWarn(customFilterListZeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") } log.Infof("intel/customlists: list loaded successful: %s", filePath) + notifications.NotifyInfo(customFilterListStatusNotificationID, + "Custom filter list loaded successfully.", + fmt.Sprintf(`Custom filter list loaded successfully from file %s +%d domains +%d IPs +%d autonomous systems +%d countries`, + filePath, + len(domainsFilterList), + len(ipAddressesFilterList), + len(autonomousSystemsFilterList), + len(domainsFilterList))) + return nil } diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 15029358..084d1b03 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -30,7 +30,8 @@ var ( filterListFilePath string filterListFileModifiedTime time.Time - parseLock sync.RWMutex + filterListLock sync.RWMutex + parserTask *modules.Task ) func init() { @@ -38,6 +39,8 @@ func init() { } func prep() error { + initFilterLists() + // register the config in the ui err := registerConfig() if err != nil { @@ -61,23 +64,29 @@ func start() error { return err } - // register timer to run every periodically and check for file updates - module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { + // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution + parserTask = module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() return nil - }).Repeat(10 * time.Minute) + }).Schedule(time.Now().Add(20 * time.Second)) - // parse the file at startup - _ = parseFile(getFilePath()) return nil } func checkAndUpdateFilterList() error { - parseLock.Lock() - defer parseLock.Unlock() + filterListLock.Lock() + defer filterListLock.Unlock() - // get path and try to get its info + // get path and ignore if empty filePath := getFilePath() + if filePath == "" { + return nil + } + + // schedule next update check + parserTask.Schedule(time.Now().Add(1 * time.Minute)) + + // try to get file info modifiedTime := time.Now() if fileInfo, err := os.Stat(filePath); err == nil { modifiedTime = fileInfo.ModTime() @@ -96,38 +105,50 @@ func checkAndUpdateFilterList() error { } // LookupIP checks if the IP address is in a custom filter list. -func LookupIP(ip *net.IP) bool { +func LookupIP(ip net.IP) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := ipAddressesFilterList[ip.String()] return ok } // LookupDomain checks if the Domain is in a custom filter list. -func LookupDomain(fullDomain string, filterSubdomains bool) bool { +func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { + filterListLock.RLock() + defer filterListLock.RUnlock() + if filterSubdomains { // check if domain is in the list and all its subdomains listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] if ok { - return true + return true, domain } } } else { // check only if the domain is in the list _, ok := domainsFilterList[fullDomain] - return ok + return ok, fullDomain } - return false + return false, "" } // LookupASN checks if the Autonomous system number is in a custom filter list. func LookupASN(number uint) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := autonomousSystemsFilterList[number] return ok } // LookupCountry checks if the country code is in a custom filter list. func LookupCountry(countryCode string) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := countryCodesFilterList[countryCode] return ok } From 3d11a1029b58c59d16430bf5841765ee5dac4d25 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Fri, 29 Jul 2022 17:37:36 +0200 Subject: [PATCH 09/15] Refactoring --- intel/customlists/config.go | 4 +- intel/customlists/lists.go | 82 ++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 33e285a9..99d5c2e6 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -11,9 +11,7 @@ var ( cfgOptionCustomListCategoryAnnotation = "Filter Lists" ) -var ( - getFilePath func() string -) +var getFilePath config.StringOption func registerConfig() error { help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index e9cbb90a..5a18d4f3 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -2,6 +2,7 @@ package customlists import ( "bufio" + "bytes" "fmt" "net" "os" @@ -22,9 +23,9 @@ var ( ) const ( - numberOfZeroIPsUntilWarning = 100 - customFilterListStatusNotificationID = "intel/customlists_status" - customFilterListZeroIPNotificationID = "intel/customlists_zeroip" + numberOfZeroIPsUntilWarning = 100 + parseStatusNotificationID = "customlists:parse-status" + zeroIPNotificationID = "customlists:too-many-zero-ips" ) func initFilterLists() { @@ -35,7 +36,7 @@ func initFilterLists() { } func parseFile(filePath string) error { - // reset all maps, previous (if any) settings will be lost + // reset all maps, previous (if any) settings will be lost. for key := range countryCodesFilterList { delete(countryCodesFilterList, key) } @@ -49,7 +50,7 @@ func parseFile(filePath string) error { delete(domainsFilterList, key) } - // ignore empty file path + // ignore empty file path. if filePath == "" { return nil } @@ -58,46 +59,35 @@ func parseFile(filePath string) error { file, err := os.Open(filePath) if err != nil { log.Warningf("intel/customlists: failed to parse file %q ", err) - // notifications.NotifyWarn("intel/customlists parse failed", "Failed to open custom filter list") - notifications.Notify(¬ifications.Notification{ - EventID: customFilterListStatusNotificationID, - Type: notifications.Warning, - Title: "Failed to open custom filter list", - Message: err.Error(), - ShowOnSystem: false, - AvailableActions: []*notifications.Action{ - { - ID: "ack", - Text: "OK", - }, - }, - }) + module.Warning(parseStatusNotificationID, "Failed to open custom filter list", err.Error()) return err } defer file.Close() var numberOfZeroIPs uint64 - // read filter file line by line + // read filter file line by line. scanner := bufio.NewScanner(file) - // the scanner will error out if the line is greater than 64K, in this case it is enough + // the scanner will error out if the line is greater than 64K, in this case it is enough. for scanner.Scan() { parseLine(scanner.Text(), &numberOfZeroIPs) } - // check for scanner error + // check for scanner error. if err := scanner.Err(); err != nil { return err } if numberOfZeroIPs >= numberOfZeroIPsUntilWarning { log.Warning("intel/customlists: Too many zero IP addresses.") - notifications.NotifyWarn(customFilterListZeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + module.Warning(zeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + } else { + module.Resolve(zeroIPNotificationID) } log.Infof("intel/customlists: list loaded successful: %s", filePath) - notifications.NotifyInfo(customFilterListStatusNotificationID, + notifications.NotifyInfo(parseStatusNotificationID, "Custom filter list loaded successfully.", fmt.Sprintf(`Custom filter list loaded successfully from file %s %d domains @@ -110,48 +100,64 @@ func parseFile(filePath string) error { len(autonomousSystemsFilterList), len(domainsFilterList))) + module.Resolve(parseStatusNotificationID) + return nil } func parseLine(line string, numberOfZeroIPs *uint64) { - // ignore empty lines and comment lines - if len(line) == 0 || line[0] == '#' { + // everything after the first field will be ignored. + fields := strings.Fields(line) + + // ignore empty lines. + if len(fields) == 0 { return } - // everything after the first field will be ignored - field := strings.Fields(line)[0] + field := fields[0] - // check if it'a a country code - if isCountryCode(field) { - countryCodesFilterList[field] = struct{}{} + // ignore comments + if field[0] == '#' { + return } - // try to parse IP address + // check if it'a a country code. + if isCountryCode(field) { + countryCodesFilterList[field] = struct{}{} + return + } + + // try to parse IP address. ip := net.ParseIP(field) if ip != nil { ipAddressesFilterList[ip.String()] = struct{}{} - // check if its zero ip - for i := 0; i < len(ip); i++ { - if ip[i] != 0 { - *numberOfZeroIPs++ + // check for zero ip. + if bytes.Compare(ip.To4(), net.IPv4zero) == 0 || bytes.Compare(ip.To16(), net.IPv6zero) == 0 { + // check if its zero ip. + for i := 0; i < len(ip); i++ { + if ip[i] != 0 { + *numberOfZeroIPs++ + } } } + return } - // check if it's a Autonomous system (example AS123) + // check if it's a Autonomous system (example AS123). if isAutonomousSystem(field) { asNumber, err := strconv.ParseUint(field[2:], 10, 32) if err != nil { return } autonomousSystemsFilterList[uint(asNumber)] = struct{}{} + return } - // check if it's a domain + // check if it's a domain. domain := dns.Fqdn(field) if netutils.IsValidFqdn(domain) { domainsFilterList[domain] = struct{}{} + return } } From fa1480d4c16d29500a59ec03aa90a08c1f9da234 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Fri, 29 Jul 2022 20:45:36 +0200 Subject: [PATCH 10/15] Change worning for invalid lines in filter list --- intel/customlists/config.go | 2 +- intel/customlists/lists.go | 58 ++++++++++++++++++++----------------- intel/customlists/module.go | 8 ++--- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 99d5c2e6..43c0cdd1 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -14,7 +14,7 @@ var ( var getFilePath config.StringOption func registerConfig() error { - help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. + help := `The file should contain list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. Lines that start with a '#' symbol are ignored. Everything after the first space/tab is ignored. Example: diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 5a18d4f3..8d47aa0c 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -23,9 +23,9 @@ var ( ) const ( - numberOfZeroIPsUntilWarning = 100 - parseStatusNotificationID = "customlists:parse-status" - zeroIPNotificationID = "customlists:too-many-zero-ips" + rationForInvalidLinesUntilWarning = 0.1 + parseStatusNotificationID = "customlists:parse-status" + zeroIPNotificationID = "customlists:too-many-zero-ips" ) func initFilterLists() { @@ -62,15 +62,20 @@ func parseFile(filePath string) error { module.Warning(parseStatusNotificationID, "Failed to open custom filter list", err.Error()) return err } - defer file.Close() + defer func() { _ = file.Close() }() - var numberOfZeroIPs uint64 + var allLinesCount uint64 + var invalidLinesCount uint64 // read filter file line by line. scanner := bufio.NewScanner(file) // the scanner will error out if the line is greater than 64K, in this case it is enough. for scanner.Scan() { - parseLine(scanner.Text(), &numberOfZeroIPs) + allLinesCount++ + // parse and count invalid lines (comment, empty lines, zero IPs...) + if !parseLine(scanner.Text()) { + invalidLinesCount++ + } } // check for scanner error. @@ -78,9 +83,13 @@ func parseFile(filePath string) error { return err } - if numberOfZeroIPs >= numberOfZeroIPsUntilWarning { - log.Warning("intel/customlists: Too many zero IP addresses.") - module.Warning(zeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + var invalidLinesRation float32 = float32(invalidLinesCount) / float32(allLinesCount) + + if invalidLinesRation > rationForInvalidLinesUntilWarning { + log.Warning("intel/customlists: Too many invalid lines") + module.Warning(zeroIPNotificationID, "Check your custom filter list, there is too many invalid lines", + fmt.Sprintf(`There are %d from total %d lines that we flagged as invalid. + Check if you are using the correct file format or if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) } else { module.Resolve(zeroIPNotificationID) } @@ -105,59 +114,56 @@ func parseFile(filePath string) error { return nil } -func parseLine(line string, numberOfZeroIPs *uint64) { +func parseLine(line string) bool { // everything after the first field will be ignored. fields := strings.Fields(line) // ignore empty lines. if len(fields) == 0 { - return + return false } field := fields[0] // ignore comments if field[0] == '#' { - return + return false } // check if it'a a country code. if isCountryCode(field) { countryCodesFilterList[field] = struct{}{} - return + return true } // try to parse IP address. ip := net.ParseIP(field) if ip != nil { - ipAddressesFilterList[ip.String()] = struct{}{} - // check for zero ip. - if bytes.Compare(ip.To4(), net.IPv4zero) == 0 || bytes.Compare(ip.To16(), net.IPv6zero) == 0 { - // check if its zero ip. - for i := 0; i < len(ip); i++ { - if ip[i] != 0 { - *numberOfZeroIPs++ - } - } + if bytes.Compare(ip, net.IPv4zero) == 0 || bytes.Compare(ip, net.IPv6zero) == 0 { + return false } - return + + ipAddressesFilterList[ip.String()] = struct{}{} + return true } // check if it's a Autonomous system (example AS123). if isAutonomousSystem(field) { asNumber, err := strconv.ParseUint(field[2:], 10, 32) if err != nil { - return + return false } autonomousSystemsFilterList[uint(asNumber)] = struct{}{} - return + return true } // check if it's a domain. domain := dns.Fqdn(field) if netutils.IsValidFqdn(domain) { domainsFilterList[domain] = struct{}{} - return + return true } + + return false } diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 084d1b03..f4e319b1 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -41,7 +41,7 @@ func init() { func prep() error { initFilterLists() - // register the config in the ui + // register the config in the ui. err := registerConfig() if err != nil { return err @@ -64,8 +64,8 @@ func start() error { return err } - // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution - parserTask = module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { + // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. + parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() return nil }).Schedule(time.Now().Add(20 * time.Second)) @@ -119,7 +119,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { defer filterListLock.RUnlock() if filterSubdomains { - // check if domain is in the list and all its subdomains + // check if domain is in the list and all its subdomains. listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] From a3c39b183ed446359ee627528b10f2a80eade152 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Mon, 1 Aug 2022 08:02:45 +0200 Subject: [PATCH 11/15] Api ednpoint to custom filter list added --- intel/customlists/module.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/intel/customlists/module.go b/intel/customlists/module.go index f4e319b1..f0cdc721 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/safing/portbase/api" "github.com/safing/portbase/modules" "golang.org/x/net/publicsuffix" ) @@ -70,6 +71,24 @@ func start() error { return nil }).Schedule(time.Now().Add(20 * time.Second)) + // register api endpoint for updating the filter list + if err := api.RegisterEndpoint(api.Endpoint{ + Path: "customlists/update", + Read: api.PermitUser, + BelongsTo: module, + ActionFunc: func(ar *api.Request) (msg string, err error) { + err = checkAndUpdateFilterList() + if err != nil { + return "failed to load custom filter list.", err + } + return "custom filter list loaded successfully.", nil + }, + Name: "Update custom filter list", + Description: "Load a filter list form a file defined by the user.", + }); err != nil { + return err + } + return nil } From d3def3ec9453f6a685bb9390937b3aaae8296f85 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Mon, 1 Aug 2022 11:24:48 +0200 Subject: [PATCH 12/15] better user messages, linter errors, refactoring --- firewall/master.go | 10 +++++----- go.mod | 2 +- go.sum | 2 ++ intel/customlists/config.go | 37 ++++++++++++++----------------------- intel/customlists/lists.go | 33 +++++++++++++++++---------------- intel/customlists/module.go | 24 ++++++++++-------------- 6 files changed, 49 insertions(+), 59 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index 50d64bc0..9ba97ced 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -619,7 +619,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the domain name appears in the custom filter list (check for subdomains if enabled) if conn.Entity.Domain != "" { if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok { - conn.Deny(fmt.Sprintf("domain %s matched %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("domain %s matches %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -628,7 +628,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi if p.FilterCNAMEs() { for _, cname := range conn.Entity.CNAME { if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok { - conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matched %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matches %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -637,7 +637,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { if customlists.LookupIP(conn.Entity.IP) { - conn.Deny(fmt.Sprintf("IP address %s appears in the custom filter list", conn.Entity.IP), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("IP address is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } @@ -645,7 +645,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block autonomous system by its number if it appears in the custom filter list if conn.Entity.ASN != 0 { if customlists.LookupASN(conn.Entity.ASN) { - conn.Deny(fmt.Sprintf("autonomous system with number %d appears in the custom filter list", conn.Entity.ASN), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("AS is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } @@ -653,7 +653,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the country appears in the custom filter list if conn.Entity.Country != "" { if customlists.LookupCountry(conn.Entity.Country) { - conn.Deny(fmt.Sprintf("country code %s appears in the custom filter list", conn.Entity.Country), customlists.CfgOptionCustomListBlockingKey) + conn.Deny("country is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) return true } } diff --git a/go.mod b/go.mod index 45638ee2..36763c35 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/miekg/dns v1.1.50 github.com/oschwald/maxminddb-golang v1.9.0 - github.com/safing/portbase v0.14.5 + github.com/safing/portbase v0.14.6 github.com/safing/spn v0.4.13 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 0b6989af..0042de61 100644 --- a/go.sum +++ b/go.sum @@ -831,6 +831,8 @@ github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc= github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= +github.com/safing/portbase v0.14.6 h1:AKj1TVJmqOrjXXWcPetchQhHVBwfD3/QDvgmkUj2FxY= +github.com/safing/portbase v0.14.6/go.mod h1:hC6KV9oZsD7yuAom42L5wgOEZ0yAjr2i1r2F5duBSA8= github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s= github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw= github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw= diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 43c0cdd1..a50c4255 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -14,34 +14,25 @@ var ( var getFilePath config.StringOption func registerConfig() error { - help := `The file should contain list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. -Lines that start with a '#' symbol are ignored. -Everything after the first space/tab is ignored. -Example: -############# -\# Domains: -example.com -google.com - -\# IP addresses -1.2.3.4 -4.3.2.1 - -\# Countries -AU -BG - -\# Autonomous Systems -AS123 -############# -> * All the records are stored in RAM, careful with large block lists. -> * Hosts files are not supported.` + help := `The file is checked every couple minutes and will be automatically reloaded when it has changed. + +Entries may be one of: +- Domain: "example.com" +- IP Address: "10.0.0.1" +- Country Code (based on IP): "US" +- AS (Autonomous System): "AS1234" + +Everything after the first element of a line, comments starting with a '#', and empty lines are ignored. +The settings "Block Subdomains of Filter List Entries" and "Block Domain Aliases" also apply to the custom filter list. +Lists in the "Hosts" format are not supported. + +Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.` // register a setting for the file path in the ui err := config.Register(&config.Option{ Name: "Custom Filter List", Key: CfgOptionCustomListBlockingKey, - Description: "Path to the file that contains a list of Domain, IP addresses, country codes and autonomous systems that you want to block", + Description: "Specify the file path to a custom filter list, which will be automatically refreshed. Any connections matching a domain, IP address, Country or ASN in the file will be blocked.", Help: help, OptType: config.OptTypeString, ExpertiseLevel: config.ExpertiseLevelExpert, diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 8d47aa0c..8b4ad7a2 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -2,7 +2,6 @@ package customlists import ( "bufio" - "bytes" "fmt" "net" "os" @@ -10,7 +9,7 @@ import ( "strings" "github.com/miekg/dns" - "github.com/safing/portbase/log" + "github.com/safing/portbase/log" //nolint // weird error "Expected '\n', Found '\t'" "github.com/safing/portbase/notifications" "github.com/safing/portmaster/network/netutils" ) @@ -25,6 +24,7 @@ var ( const ( rationForInvalidLinesUntilWarning = 0.1 parseStatusNotificationID = "customlists:parse-status" + parseWarningNotificationID = "customlists:parse-warning" zeroIPNotificationID = "customlists:too-many-zero-ips" ) @@ -58,8 +58,8 @@ func parseFile(filePath string) error { // open the file if possible file, err := os.Open(filePath) if err != nil { - log.Warningf("intel/customlists: failed to parse file %q ", err) - module.Warning(parseStatusNotificationID, "Failed to open custom filter list", err.Error()) + log.Warningf("intel/customlists: failed to parse file %s", err) + module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error()) return err } defer func() { _ = file.Close() }() @@ -83,33 +83,34 @@ func parseFile(filePath string) error { return err } - var invalidLinesRation float32 = float32(invalidLinesCount) / float32(allLinesCount) + invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount) if invalidLinesRation > rationForInvalidLinesUntilWarning { log.Warning("intel/customlists: Too many invalid lines") - module.Warning(zeroIPNotificationID, "Check your custom filter list, there is too many invalid lines", - fmt.Sprintf(`There are %d from total %d lines that we flagged as invalid. - Check if you are using the correct file format or if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) + module.Warning(zeroIPNotificationID, "Custom filter list has many invalid entries", + fmt.Sprintf(`%d out of %d entires are invalid. + Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) } else { module.Resolve(zeroIPNotificationID) } - log.Infof("intel/customlists: list loaded successful: %s", filePath) + allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList) + log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath) notifications.NotifyInfo(parseStatusNotificationID, "Custom filter list loaded successfully.", - fmt.Sprintf(`Custom filter list loaded successfully from file %s -%d domains + fmt.Sprintf(`Custom filter list loaded successfully from file %s - loaded: +%d Domains %d IPs -%d autonomous systems -%d countries`, +%d Autonomous Systems +%d Countries`, filePath, len(domainsFilterList), len(ipAddressesFilterList), len(autonomousSystemsFilterList), - len(domainsFilterList))) + len(countryCodesFilterList))) - module.Resolve(parseStatusNotificationID) + module.Resolve(parseWarningNotificationID) return nil } @@ -140,7 +141,7 @@ func parseLine(line string) bool { ip := net.ParseIP(field) if ip != nil { // check for zero ip. - if bytes.Compare(ip, net.IPv4zero) == 0 || bytes.Compare(ip, net.IPv6zero) == 0 { + if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { return false } diff --git a/intel/customlists/module.go b/intel/customlists/module.go index f0cdc721..fd918c60 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/safing/portbase/api" + "github.com/safing/portbase/api" //nolint // completing about import order!? "github.com/safing/portbase/modules" "golang.org/x/net/publicsuffix" ) @@ -58,7 +58,7 @@ func start() error { configChangeEvent, "update custom filter list", func(ctx context.Context, obj interface{}) error { - _ = checkAndUpdateFilterList() + checkAndUpdateFilterList() return nil }, ); err != nil { @@ -67,24 +67,21 @@ func start() error { // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { - _ = checkAndUpdateFilterList() + checkAndUpdateFilterList() return nil }).Schedule(time.Now().Add(20 * time.Second)) // register api endpoint for updating the filter list if err := api.RegisterEndpoint(api.Endpoint{ Path: "customlists/update", - Read: api.PermitUser, + Write: api.PermitUser, BelongsTo: module, ActionFunc: func(ar *api.Request) (msg string, err error) { - err = checkAndUpdateFilterList() - if err != nil { - return "failed to load custom filter list.", err - } - return "custom filter list loaded successfully.", nil + checkAndUpdateFilterList() + return "Custom filter list loaded successfully.", nil }, Name: "Update custom filter list", - Description: "Load a filter list form a file defined by the user.", + Description: "Reload the filter list from the configured file.", }); err != nil { return err } @@ -92,14 +89,14 @@ func start() error { return nil } -func checkAndUpdateFilterList() error { +func checkAndUpdateFilterList() { filterListLock.Lock() defer filterListLock.Unlock() // get path and ignore if empty filePath := getFilePath() if filePath == "" { - return nil + return } // schedule next update check @@ -115,12 +112,11 @@ func checkAndUpdateFilterList() error { if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { err := parseFile(filePath) if err != nil { - return nil + return } filterListFileModifiedTime = modifiedTime filterListFilePath = filePath } - return nil } // LookupIP checks if the IP address is in a custom filter list. From cc711e477d0a2842d44a63598b1bc9897f3d424e Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Mon, 1 Aug 2022 11:34:08 +0200 Subject: [PATCH 13/15] fix linter error --- intel/customlists/lists.go | 3 ++- intel/customlists/module.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 8b4ad7a2..51907f06 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -9,7 +9,8 @@ import ( "strings" "github.com/miekg/dns" - "github.com/safing/portbase/log" //nolint // weird error "Expected '\n', Found '\t'" + + "github.com/safing/portbase/log" "github.com/safing/portbase/notifications" "github.com/safing/portmaster/network/netutils" ) diff --git a/intel/customlists/module.go b/intel/customlists/module.go index fd918c60..e1d187e0 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -9,9 +9,10 @@ import ( "sync" "time" - "github.com/safing/portbase/api" //nolint // completing about import order!? - "github.com/safing/portbase/modules" "golang.org/x/net/publicsuffix" + + "github.com/safing/portbase/api" + "github.com/safing/portbase/modules" ) var module *modules.Module From 549fa1926fe9a5a4f42c513517f38a5844af1adc Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 1 Aug 2022 14:10:28 +0200 Subject: [PATCH 14/15] Move checking custom filter list behind main filter lists to get entity data --- firewall/master.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firewall/master.go b/firewall/master.go index 9ba97ced..abe9d017 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -51,8 +51,8 @@ var defaultDeciders = []deciderFn{ checkResolverScope, checkConnectivityDomain, checkBypassPrevention, - checkCustomFilterList, checkFilterLists, + checkCustomFilterList, dropInbound, checkDomainHeuristics, checkAutoPermitRelated, From ced6690acdf77dabbe922b90b1f1f910e719073c Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 1 Aug 2022 14:11:01 +0200 Subject: [PATCH 15/15] Minor improvements and documentation --- intel/customlists/config.go | 2 +- intel/customlists/lists.go | 67 ++++++++++++++++++++----------------- intel/customlists/module.go | 20 +++++------ 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/intel/customlists/config.go b/intel/customlists/config.go index a50c4255..f58f2f67 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -28,7 +28,7 @@ Lists in the "Hosts" format are not supported. Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.` - // register a setting for the file path in the ui + // Register a setting for the file path in the ui err := config.Register(&config.Option{ Name: "Custom Filter List", Key: CfgOptionCustomListBlockingKey, diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 51907f06..81bb8929 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -37,7 +37,7 @@ func initFilterLists() { } func parseFile(filePath string) error { - // reset all maps, previous (if any) settings will be lost. + // Reset all maps, previous (if any) settings will be lost. for key := range countryCodesFilterList { delete(countryCodesFilterList, key) } @@ -51,12 +51,12 @@ func parseFile(filePath string) error { delete(domainsFilterList, key) } - // ignore empty file path. + // Ignore empty file path. if filePath == "" { return nil } - // open the file if possible + // Open the file if possible file, err := os.Open(filePath) if err != nil { log.Warningf("intel/customlists: failed to parse file %s", err) @@ -68,18 +68,18 @@ func parseFile(filePath string) error { var allLinesCount uint64 var invalidLinesCount uint64 - // read filter file line by line. + // Read filter file line by line. scanner := bufio.NewScanner(file) - // the scanner will error out if the line is greater than 64K, in this case it is enough. + // The scanner will error out if the line is greater than 64K, in this case it is enough. for scanner.Scan() { allLinesCount++ - // parse and count invalid lines (comment, empty lines, zero IPs...) + // Parse and count invalid lines (comment, empty lines, zero IPs...) if !parseLine(scanner.Text()) { invalidLinesCount++ } } - // check for scanner error. + // Check for scanner error. if err := scanner.Err(); err != nil { return err } @@ -88,8 +88,8 @@ func parseFile(filePath string) error { if invalidLinesRation > rationForInvalidLinesUntilWarning { log.Warning("intel/customlists: Too many invalid lines") - module.Warning(zeroIPNotificationID, "Custom filter list has many invalid entries", - fmt.Sprintf(`%d out of %d entires are invalid. + module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines", + fmt.Sprintf(`%d out of %d lines are invalid. Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) } else { module.Resolve(zeroIPNotificationID) @@ -116,41 +116,34 @@ func parseFile(filePath string) error { return nil } -func parseLine(line string) bool { - // everything after the first field will be ignored. +func parseLine(line string) (valid bool) { + // Everything after the first field will be ignored. fields := strings.Fields(line) - // ignore empty lines. + // Ignore empty lines. if len(fields) == 0 { - return false + return true // Not an entry, but a valid line. } field := fields[0] - // ignore comments - if field[0] == '#' { - return false + // Ignore comments + if strings.HasPrefix(field, "#") { + return true // Not an entry, but a valid line. } - // check if it'a a country code. + // Go through all possible field types. + // Parsing is ordered by + // 1. Parsing options (ie. the domain has most variation and goes last.) + // 2. Speed + + // Check if it'a a country code. if isCountryCode(field) { countryCodesFilterList[field] = struct{}{} return true } - // try to parse IP address. - ip := net.ParseIP(field) - if ip != nil { - // check for zero ip. - if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { - return false - } - - ipAddressesFilterList[ip.String()] = struct{}{} - return true - } - - // check if it's a Autonomous system (example AS123). + // Check if it's a Autonomous system (example AS123). if isAutonomousSystem(field) { asNumber, err := strconv.ParseUint(field[2:], 10, 32) if err != nil { @@ -160,7 +153,19 @@ func parseLine(line string) bool { return true } - // check if it's a domain. + // Try to parse IP address. + ip := net.ParseIP(field) + if ip != nil { + // Check for zero ip. + if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { + return false + } + + ipAddressesFilterList[ip.String()] = struct{}{} + return true + } + + // Check if it's a domain. domain := dns.Fqdn(field) if netutils.IsValidFqdn(domain) { domainsFilterList[domain] = struct{}{} diff --git a/intel/customlists/module.go b/intel/customlists/module.go index e1d187e0..e20cc890 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -43,7 +43,7 @@ func init() { func prep() error { initFilterLists() - // register the config in the ui. + // Register the config in the ui. err := registerConfig() if err != nil { return err @@ -53,7 +53,7 @@ func prep() error { } func start() error { - // register to hook to update after config change. + // Register to hook to update after config change. if err := module.RegisterEventHook( configModuleName, configChangeEvent, @@ -66,13 +66,13 @@ func start() error { return err } - // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. + // Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { checkAndUpdateFilterList() return nil }).Schedule(time.Now().Add(20 * time.Second)) - // register api endpoint for updating the filter list + // Register api endpoint for updating the filter list. if err := api.RegisterEndpoint(api.Endpoint{ Path: "customlists/update", Write: api.PermitUser, @@ -94,22 +94,22 @@ func checkAndUpdateFilterList() { filterListLock.Lock() defer filterListLock.Unlock() - // get path and ignore if empty + // Get path and ignore if empty filePath := getFilePath() if filePath == "" { return } - // schedule next update check + // Schedule next update check parserTask.Schedule(time.Now().Add(1 * time.Minute)) - // try to get file info + // Try to get file info modifiedTime := time.Now() if fileInfo, err := os.Stat(filePath); err == nil { modifiedTime = fileInfo.ModTime() } - // check if file path has changed or if modified time has changed + // Check if file path has changed or if modified time has changed if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { err := parseFile(filePath) if err != nil { @@ -135,7 +135,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { defer filterListLock.RUnlock() if filterSubdomains { - // check if domain is in the list and all its subdomains. + // Check if domain is in the list and all its subdomains. listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] @@ -144,7 +144,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { } } } else { - // check only if the domain is in the list + // Check only if the domain is in the list _, ok := domainsFilterList[fullDomain] return ok, fullDomain }