diff --git a/firewall/master.go b/firewall/master.go index 1de7f194..abe9d017 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -13,6 +13,7 @@ import ( "github.com/safing/portbase/log" "github.com/safing/portmaster/detection/dga" + "github.com/safing/portmaster/intel/customlists" "github.com/safing/portmaster/netenv" "github.com/safing/portmaster/network" "github.com/safing/portmaster/network/netutils" @@ -51,6 +52,7 @@ var defaultDeciders = []deciderFn{ checkConnectivityDomain, checkBypassPrevention, checkFilterLists, + checkCustomFilterList, dropInbound, checkDomainHeuristics, checkAutoPermitRelated, @@ -612,3 +614,49 @@ matchLoop: } return related, reason } + +func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { + // block if the domain name appears in the custom filter list (check for subdomains if enabled) + if conn.Entity.Domain != "" { + if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain %s matches %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + // block if any of the CNAME appears in the custom filter list (check for subdomains if enabled) + if p.FilterCNAMEs() { + for _, cname := range conn.Entity.CNAME { + if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matches %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) + return true + } + } + } + + // block if ip addresses appears in the custom filter list + if conn.Entity.IP != nil { + if customlists.LookupIP(conn.Entity.IP) { + conn.Deny("IP address is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + // block autonomous system by its number if it appears in the custom filter list + if conn.Entity.ASN != 0 { + if customlists.LookupASN(conn.Entity.ASN) { + conn.Deny("AS is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + // block if the country appears in the custom filter list + if conn.Entity.Country != "" { + if customlists.LookupCountry(conn.Entity.Country) { + conn.Deny("country is in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + return true + } + } + + return false +} diff --git a/go.mod b/go.mod index 45638ee2..36763c35 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/miekg/dns v1.1.50 github.com/oschwald/maxminddb-golang v1.9.0 - github.com/safing/portbase v0.14.5 + github.com/safing/portbase v0.14.6 github.com/safing/spn v0.4.13 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 0b6989af..0042de61 100644 --- a/go.sum +++ b/go.sum @@ -831,6 +831,8 @@ github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc= github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= +github.com/safing/portbase v0.14.6 h1:AKj1TVJmqOrjXXWcPetchQhHVBwfD3/QDvgmkUj2FxY= +github.com/safing/portbase v0.14.6/go.mod h1:hC6KV9oZsD7yuAom42L5wgOEZ0yAjr2i1r2F5duBSA8= github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s= github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw= github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw= diff --git a/intel/customlists/config.go b/intel/customlists/config.go new file mode 100644 index 00000000..f58f2f67 --- /dev/null +++ b/intel/customlists/config.go @@ -0,0 +1,55 @@ +package customlists + +import ( + "github.com/safing/portbase/config" +) + +var ( + // CfgOptionCustomListBlockingKey is the config key for the listen address.. + CfgOptionCustomListBlockingKey = "filter/customListBlocking" + cfgOptionCustomListBlockingOrder = 35 + cfgOptionCustomListCategoryAnnotation = "Filter Lists" +) + +var getFilePath config.StringOption + +func registerConfig() error { + help := `The file is checked every couple minutes and will be automatically reloaded when it has changed. + +Entries may be one of: +- Domain: "example.com" +- IP Address: "10.0.0.1" +- Country Code (based on IP): "US" +- AS (Autonomous System): "AS1234" + +Everything after the first element of a line, comments starting with a '#', and empty lines are ignored. +The settings "Block Subdomains of Filter List Entries" and "Block Domain Aliases" also apply to the custom filter list. +Lists in the "Hosts" format are not supported. + +Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.` + + // Register a setting for the file path in the ui + err := config.Register(&config.Option{ + Name: "Custom Filter List", + Key: CfgOptionCustomListBlockingKey, + Description: "Specify the file path to a custom filter list, which will be automatically refreshed. Any connections matching a domain, IP address, Country or ASN in the file will be blocked.", + Help: help, + OptType: config.OptTypeString, + ExpertiseLevel: config.ExpertiseLevelExpert, + ReleaseLevel: config.ReleaseLevelStable, + DefaultValue: "", + RequiresRestart: false, + Annotations: config.Annotations{ + config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder, + config.CategoryAnnotation: cfgOptionCustomListCategoryAnnotation, + config.DisplayHintAnnotation: config.DisplayHintFilePicker, + }, + }) + if err != nil { + return err + } + + getFilePath = config.GetAsString(CfgOptionCustomListBlockingKey, "") + + return nil +} diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go new file mode 100644 index 00000000..81bb8929 --- /dev/null +++ b/intel/customlists/lists.go @@ -0,0 +1,176 @@ +package customlists + +import ( + "bufio" + "fmt" + "net" + "os" + "strconv" + "strings" + + "github.com/miekg/dns" + + "github.com/safing/portbase/log" + "github.com/safing/portbase/notifications" + "github.com/safing/portmaster/network/netutils" +) + +var ( + countryCodesFilterList map[string]struct{} + ipAddressesFilterList map[string]struct{} + autonomousSystemsFilterList map[uint]struct{} + domainsFilterList map[string]struct{} +) + +const ( + rationForInvalidLinesUntilWarning = 0.1 + parseStatusNotificationID = "customlists:parse-status" + parseWarningNotificationID = "customlists:parse-warning" + zeroIPNotificationID = "customlists:too-many-zero-ips" +) + +func initFilterLists() { + countryCodesFilterList = make(map[string]struct{}) + ipAddressesFilterList = make(map[string]struct{}) + autonomousSystemsFilterList = make(map[uint]struct{}) + domainsFilterList = make(map[string]struct{}) +} + +func parseFile(filePath string) error { + // Reset all maps, previous (if any) settings will be lost. + for key := range countryCodesFilterList { + delete(countryCodesFilterList, key) + } + for key := range ipAddressesFilterList { + delete(ipAddressesFilterList, key) + } + for key := range autonomousSystemsFilterList { + delete(autonomousSystemsFilterList, key) + } + for key := range domainsFilterList { + delete(domainsFilterList, key) + } + + // Ignore empty file path. + if filePath == "" { + return nil + } + + // Open the file if possible + file, err := os.Open(filePath) + if err != nil { + log.Warningf("intel/customlists: failed to parse file %s", err) + module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error()) + return err + } + defer func() { _ = file.Close() }() + + var allLinesCount uint64 + var invalidLinesCount uint64 + + // Read filter file line by line. + scanner := bufio.NewScanner(file) + // The scanner will error out if the line is greater than 64K, in this case it is enough. + for scanner.Scan() { + allLinesCount++ + // Parse and count invalid lines (comment, empty lines, zero IPs...) + if !parseLine(scanner.Text()) { + invalidLinesCount++ + } + } + + // Check for scanner error. + if err := scanner.Err(); err != nil { + return err + } + + invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount) + + if invalidLinesRation > rationForInvalidLinesUntilWarning { + log.Warning("intel/customlists: Too many invalid lines") + module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines", + fmt.Sprintf(`%d out of %d lines are invalid. + Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) + } else { + module.Resolve(zeroIPNotificationID) + } + + allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList) + log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath) + + notifications.NotifyInfo(parseStatusNotificationID, + "Custom filter list loaded successfully.", + fmt.Sprintf(`Custom filter list loaded successfully from file %s - loaded: +%d Domains +%d IPs +%d Autonomous Systems +%d Countries`, + filePath, + len(domainsFilterList), + len(ipAddressesFilterList), + len(autonomousSystemsFilterList), + len(countryCodesFilterList))) + + module.Resolve(parseWarningNotificationID) + + return nil +} + +func parseLine(line string) (valid bool) { + // Everything after the first field will be ignored. + fields := strings.Fields(line) + + // Ignore empty lines. + if len(fields) == 0 { + return true // Not an entry, but a valid line. + } + + field := fields[0] + + // Ignore comments + if strings.HasPrefix(field, "#") { + return true // Not an entry, but a valid line. + } + + // Go through all possible field types. + // Parsing is ordered by + // 1. Parsing options (ie. the domain has most variation and goes last.) + // 2. Speed + + // Check if it'a a country code. + if isCountryCode(field) { + countryCodesFilterList[field] = struct{}{} + return true + } + + // Check if it's a Autonomous system (example AS123). + if isAutonomousSystem(field) { + asNumber, err := strconv.ParseUint(field[2:], 10, 32) + if err != nil { + return false + } + autonomousSystemsFilterList[uint(asNumber)] = struct{}{} + return true + } + + // Try to parse IP address. + ip := net.ParseIP(field) + if ip != nil { + // Check for zero ip. + if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { + return false + } + + ipAddressesFilterList[ip.String()] = struct{}{} + return true + } + + // Check if it's a domain. + domain := dns.Fqdn(field) + if netutils.IsValidFqdn(domain) { + domainsFilterList[domain] = struct{}{} + return true + } + + return false +} diff --git a/intel/customlists/module.go b/intel/customlists/module.go new file mode 100644 index 00000000..e20cc890 --- /dev/null +++ b/intel/customlists/module.go @@ -0,0 +1,196 @@ +package customlists + +import ( + "context" + "net" + "os" + "regexp" + "strings" + "sync" + "time" + + "golang.org/x/net/publicsuffix" + + "github.com/safing/portbase/api" + "github.com/safing/portbase/modules" +) + +var module *modules.Module + +const ( + configModuleName = "config" + configChangeEvent = "config change" +) + +// Helper variables for parsing the input file. +var ( + isCountryCode = regexp.MustCompile("^[A-Z]{2}$").MatchString + isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString +) + +var ( + filterListFilePath string + filterListFileModifiedTime time.Time + + filterListLock sync.RWMutex + parserTask *modules.Task +) + +func init() { + module = modules.Register("customlists", prep, start, nil, "base") +} + +func prep() error { + initFilterLists() + + // Register the config in the ui. + err := registerConfig() + if err != nil { + return err + } + + return nil +} + +func start() error { + // Register to hook to update after config change. + if err := module.RegisterEventHook( + configModuleName, + configChangeEvent, + "update custom filter list", + func(ctx context.Context, obj interface{}) error { + checkAndUpdateFilterList() + return nil + }, + ); err != nil { + return err + } + + // Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. + parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { + checkAndUpdateFilterList() + return nil + }).Schedule(time.Now().Add(20 * time.Second)) + + // Register api endpoint for updating the filter list. + if err := api.RegisterEndpoint(api.Endpoint{ + Path: "customlists/update", + Write: api.PermitUser, + BelongsTo: module, + ActionFunc: func(ar *api.Request) (msg string, err error) { + checkAndUpdateFilterList() + return "Custom filter list loaded successfully.", nil + }, + Name: "Update custom filter list", + Description: "Reload the filter list from the configured file.", + }); err != nil { + return err + } + + return nil +} + +func checkAndUpdateFilterList() { + filterListLock.Lock() + defer filterListLock.Unlock() + + // Get path and ignore if empty + filePath := getFilePath() + if filePath == "" { + return + } + + // Schedule next update check + parserTask.Schedule(time.Now().Add(1 * time.Minute)) + + // Try to get file info + modifiedTime := time.Now() + if fileInfo, err := os.Stat(filePath); err == nil { + modifiedTime = fileInfo.ModTime() + } + + // Check if file path has changed or if modified time has changed + if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { + err := parseFile(filePath) + if err != nil { + return + } + filterListFileModifiedTime = modifiedTime + filterListFilePath = filePath + } +} + +// LookupIP checks if the IP address is in a custom filter list. +func LookupIP(ip net.IP) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + + _, ok := ipAddressesFilterList[ip.String()] + return ok +} + +// LookupDomain checks if the Domain is in a custom filter list. +func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { + filterListLock.RLock() + defer filterListLock.RUnlock() + + if filterSubdomains { + // Check if domain is in the list and all its subdomains. + listOfDomains := splitDomain(fullDomain) + for _, domain := range listOfDomains { + _, ok := domainsFilterList[domain] + if ok { + return true, domain + } + } + } else { + // Check only if the domain is in the list + _, ok := domainsFilterList[fullDomain] + return ok, fullDomain + } + return false, "" +} + +// LookupASN checks if the Autonomous system number is in a custom filter list. +func LookupASN(number uint) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + + _, ok := autonomousSystemsFilterList[number] + return ok +} + +// LookupCountry checks if the country code is in a custom filter list. +func LookupCountry(countryCode string) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + + _, ok := countryCodesFilterList[countryCode] + return ok +} + +func splitDomain(domain string) []string { + domain = strings.Trim(domain, ".") + suffix, _ := publicsuffix.PublicSuffix(domain) + if suffix == domain { + return []string{domain} + } + + domainWithoutSuffix := domain[:len(domain)-len(suffix)] + domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".") + + splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool { + return r == '.' + }) + + domains := make([]string, 0, len(splitted)) + for idx := range splitted { + + d := strings.Join(splitted[idx:], ".") + "." + suffix + if d[len(d)-1] != '.' { + d += "." + } + domains = append(domains, d) + } + return domains +} diff --git a/intel/module.go b/intel/module.go index 65fcc65d..ceec6b64 100644 --- a/intel/module.go +++ b/intel/module.go @@ -2,11 +2,12 @@ package intel import ( "github.com/safing/portbase/modules" + _ "github.com/safing/portmaster/intel/customlists" ) // Module of this package. Export needed for testing of the endpoints package. var Module *modules.Module func init() { - Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists") + Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists", "customlists") } diff --git a/profile/config.go b/profile/config.go index e371a175..8963d2ad 100644 --- a/profile/config.go +++ b/profile/config.go @@ -64,9 +64,11 @@ var ( cfgOptionFilterLists config.StringArrayOption cfgOptionFilterListsOrder = 34 + // Setting "Custom Filter List" at order 35. + CfgOptionFilterSubDomainsKey = "filter/includeSubdomains" cfgOptionFilterSubDomains config.IntOption // security level option - cfgOptionFilterSubDomainsOrder = 35 + cfgOptionFilterSubDomainsOrder = 36 // DNS Filtering.