Merge pull request #761 from safing/feature/custom-filter-list

Feature: Custom filter list

firewall/master.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/safing/portbase/log"
 	"github.com/safing/portmaster/detection/dga"
+	"github.com/safing/portmaster/intel/customlists"
 	"github.com/safing/portmaster/netenv"
 	"github.com/safing/portmaster/network"
 	"github.com/safing/portmaster/network/netutils"
@@ -51,6 +52,7 @@ var defaultDeciders = []deciderFn{
 	checkConnectivityDomain,
 	checkBypassPrevention,
 	checkFilterLists,
+	checkCustomFilterList,
 	dropInbound,
 	checkDomainHeuristics,
 	checkAutoPermitRelated,
@@ -612,3 +614,49 @@ matchLoop:
 	}
 	return related, reason
 }
+
+func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
+	// block if the domain name appears in the custom filter list (check for subdomains if enabled)
+	if conn.Entity.Domain != "" {
+		if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok {
+			conn.Deny(fmt.Sprintf("domain %s matches %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey)
+			return true
+		}
+	}
+
+	// block if any of the CNAME appears in the custom filter list (check for subdomains if enabled)
+	if p.FilterCNAMEs() {
+		for _, cname := range conn.Entity.CNAME {
+			if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok {
+				conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matches %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey)
+				return true
+			}
+		}
+	}
+
+	// block if ip addresses appears in the custom filter list
+	if conn.Entity.IP != nil {
+		if customlists.LookupIP(conn.Entity.IP) {
+			conn.Deny("IP address is in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			return true
+		}
+	}
+
+	// block autonomous system by its number if it appears in the custom filter list
+	if conn.Entity.ASN != 0 {
+		if customlists.LookupASN(conn.Entity.ASN) {
+			conn.Deny("AS is in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			return true
+		}
+	}
+
+	// block if the country appears in the custom filter list
+	if conn.Entity.Country != "" {
+		if customlists.LookupCountry(conn.Entity.Country) {
+			conn.Deny("country is in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			return true
+		}
+	}
+
+	return false
+}

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/miekg/dns v1.1.50
 	github.com/oschwald/maxminddb-golang v1.9.0
-	github.com/safing/portbase v0.14.5
+	github.com/safing/portbase v0.14.6
 	github.com/safing/spn v0.4.13
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.5.0

go.sum

@@ -831,6 +831,8 @@ github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X
 github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc=
 github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.6 h1:AKj1TVJmqOrjXXWcPetchQhHVBwfD3/QDvgmkUj2FxY=
+github.com/safing/portbase v0.14.6/go.mod h1:hC6KV9oZsD7yuAom42L5wgOEZ0yAjr2i1r2F5duBSA8=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=

intel/customlists/config.go

@@ -0,0 +1,55 @@
+package customlists
+
+import (
+	"github.com/safing/portbase/config"
+)
+
+var (
+	// CfgOptionCustomListBlockingKey is the config key for the listen address..
+	CfgOptionCustomListBlockingKey        = "filter/customListBlocking"
+	cfgOptionCustomListBlockingOrder      = 35
+	cfgOptionCustomListCategoryAnnotation = "Filter Lists"
+)
+
+var getFilePath config.StringOption
+
+func registerConfig() error {
+	help := `The file is checked every couple minutes and will be automatically reloaded when it has changed.  
+
+Entries may be one of:
+- Domain: "example.com"
+- IP Address: "10.0.0.1"
+- Country Code (based on IP): "US"
+- AS (Autonomous System): "AS1234"  
+
+Everything after the first element of a line, comments starting with a '#', and empty lines are ignored.  
+The settings "Block Subdomains of Filter List Entries" and "Block Domain Aliases" also apply to the custom filter list.  
+Lists in the "Hosts" format are not supported.  
+
+Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.`
+
+	// Register a setting for the file path in the ui
+	err := config.Register(&config.Option{
+		Name:            "Custom Filter List",
+		Key:             CfgOptionCustomListBlockingKey,
+		Description:     "Specify the file path to a custom filter list, which will be automatically refreshed. Any connections matching a domain, IP address, Country or ASN in the file will be blocked.",
+		Help:            help,
+		OptType:         config.OptTypeString,
+		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ReleaseLevel:    config.ReleaseLevelStable,
+		DefaultValue:    "",
+		RequiresRestart: false,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder,
+			config.CategoryAnnotation:     cfgOptionCustomListCategoryAnnotation,
+			config.DisplayHintAnnotation:  config.DisplayHintFilePicker,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	getFilePath = config.GetAsString(CfgOptionCustomListBlockingKey, "")
+
+	return nil
+}

intel/customlists/lists.go

@@ -0,0 +1,176 @@
+package customlists
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/miekg/dns"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/notifications"
+	"github.com/safing/portmaster/network/netutils"
+)
+
+var (
+	countryCodesFilterList      map[string]struct{}
+	ipAddressesFilterList       map[string]struct{}
+	autonomousSystemsFilterList map[uint]struct{}
+	domainsFilterList           map[string]struct{}
+)
+
+const (
+	rationForInvalidLinesUntilWarning = 0.1
+	parseStatusNotificationID         = "customlists:parse-status"
+	parseWarningNotificationID        = "customlists:parse-warning"
+	zeroIPNotificationID              = "customlists:too-many-zero-ips"
+)
+
+func initFilterLists() {
+	countryCodesFilterList = make(map[string]struct{})
+	ipAddressesFilterList = make(map[string]struct{})
+	autonomousSystemsFilterList = make(map[uint]struct{})
+	domainsFilterList = make(map[string]struct{})
+}
+
+func parseFile(filePath string) error {
+	// Reset all maps, previous (if any) settings will be lost.
+	for key := range countryCodesFilterList {
+		delete(countryCodesFilterList, key)
+	}
+	for key := range ipAddressesFilterList {
+		delete(ipAddressesFilterList, key)
+	}
+	for key := range autonomousSystemsFilterList {
+		delete(autonomousSystemsFilterList, key)
+	}
+	for key := range domainsFilterList {
+		delete(domainsFilterList, key)
+	}
+
+	// Ignore empty file path.
+	if filePath == "" {
+		return nil
+	}
+
+	// Open the file if possible
+	file, err := os.Open(filePath)
+	if err != nil {
+		log.Warningf("intel/customlists: failed to parse file %s", err)
+		module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error())
+		return err
+	}
+	defer func() { _ = file.Close() }()
+
+	var allLinesCount uint64
+	var invalidLinesCount uint64
+
+	// Read filter file line by line.
+	scanner := bufio.NewScanner(file)
+	// The scanner will error out if the line is greater than 64K, in this case it is enough.
+	for scanner.Scan() {
+		allLinesCount++
+		// Parse and count invalid lines (comment, empty lines, zero IPs...)
+		if !parseLine(scanner.Text()) {
+			invalidLinesCount++
+		}
+	}
+
+	// Check for scanner error.
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount)
+
+	if invalidLinesRation > rationForInvalidLinesUntilWarning {
+		log.Warning("intel/customlists: Too many invalid lines")
+		module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines",
+			fmt.Sprintf(`%d out of %d lines are invalid.
+			 Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount))
+	} else {
+		module.Resolve(zeroIPNotificationID)
+	}
+
+	allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList)
+	log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath)
+
+	notifications.NotifyInfo(parseStatusNotificationID,
+		"Custom filter list loaded successfully.",
+		fmt.Sprintf(`Custom filter list loaded successfully from file %s - loaded:  
+%d Domains  
+%d IPs  
+%d Autonomous Systems  
+%d Countries`,
+			filePath,
+			len(domainsFilterList),
+			len(ipAddressesFilterList),
+			len(autonomousSystemsFilterList),
+			len(countryCodesFilterList)))
+
+	module.Resolve(parseWarningNotificationID)
+
+	return nil
+}
+
+func parseLine(line string) (valid bool) {
+	// Everything after the first field will be ignored.
+	fields := strings.Fields(line)
+
+	// Ignore empty lines.
+	if len(fields) == 0 {
+		return true // Not an entry, but a valid line.
+	}
+
+	field := fields[0]
+
+	// Ignore comments
+	if strings.HasPrefix(field, "#") {
+		return true // Not an entry, but a valid line.
+	}
+
+	// Go through all possible field types.
+	// Parsing is ordered by
+	// 1. Parsing options (ie. the domain has most variation and goes last.)
+	// 2. Speed
+
+	// Check if it'a a country code.
+	if isCountryCode(field) {
+		countryCodesFilterList[field] = struct{}{}
+		return true
+	}
+
+	// Check if it's a Autonomous system (example AS123).
+	if isAutonomousSystem(field) {
+		asNumber, err := strconv.ParseUint(field[2:], 10, 32)
+		if err != nil {
+			return false
+		}
+		autonomousSystemsFilterList[uint(asNumber)] = struct{}{}
+		return true
+	}
+
+	// Try to parse IP address.
+	ip := net.ParseIP(field)
+	if ip != nil {
+		// Check for zero ip.
+		if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) {
+			return false
+		}
+
+		ipAddressesFilterList[ip.String()] = struct{}{}
+		return true
+	}
+
+	// Check if it's a domain.
+	domain := dns.Fqdn(field)
+	if netutils.IsValidFqdn(domain) {
+		domainsFilterList[domain] = struct{}{}
+		return true
+	}
+
+	return false
+}

intel/customlists/module.go

@@ -0,0 +1,196 @@
+package customlists
+
+import (
+	"context"
+	"net"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/publicsuffix"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/modules"
+)
+
+var module *modules.Module
+
+const (
+	configModuleName  = "config"
+	configChangeEvent = "config change"
+)
+
+// Helper variables for parsing the input file.
+var (
+	isCountryCode      = regexp.MustCompile("^[A-Z]{2}$").MatchString
+	isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString
+)
+
+var (
+	filterListFilePath         string
+	filterListFileModifiedTime time.Time
+
+	filterListLock sync.RWMutex
+	parserTask     *modules.Task
+)
+
+func init() {
+	module = modules.Register("customlists", prep, start, nil, "base")
+}
+
+func prep() error {
+	initFilterLists()
+
+	// Register the config in the ui.
+	err := registerConfig()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func start() error {
+	// Register to hook to update after config change.
+	if err := module.RegisterEventHook(
+		configModuleName,
+		configChangeEvent,
+		"update custom filter list",
+		func(ctx context.Context, obj interface{}) error {
+			checkAndUpdateFilterList()
+			return nil
+		},
+	); err != nil {
+		return err
+	}
+
+	// Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
+	parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error {
+		checkAndUpdateFilterList()
+		return nil
+	}).Schedule(time.Now().Add(20 * time.Second))
+
+	// Register api endpoint for updating the filter list.
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:      "customlists/update",
+		Write:     api.PermitUser,
+		BelongsTo: module,
+		ActionFunc: func(ar *api.Request) (msg string, err error) {
+			checkAndUpdateFilterList()
+			return "Custom filter list loaded successfully.", nil
+		},
+		Name:        "Update custom filter list",
+		Description: "Reload the filter list from the configured file.",
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkAndUpdateFilterList() {
+	filterListLock.Lock()
+	defer filterListLock.Unlock()
+
+	// Get path and ignore if empty
+	filePath := getFilePath()
+	if filePath == "" {
+		return
+	}
+
+	// Schedule next update check
+	parserTask.Schedule(time.Now().Add(1 * time.Minute))
+
+	// Try to get file info
+	modifiedTime := time.Now()
+	if fileInfo, err := os.Stat(filePath); err == nil {
+		modifiedTime = fileInfo.ModTime()
+	}
+
+	// Check if file path has changed or if modified time has changed
+	if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) {
+		err := parseFile(filePath)
+		if err != nil {
+			return
+		}
+		filterListFileModifiedTime = modifiedTime
+		filterListFilePath = filePath
+	}
+}
+
+// LookupIP checks if the IP address is in a custom filter list.
+func LookupIP(ip net.IP) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := ipAddressesFilterList[ip.String()]
+	return ok
+}
+
+// LookupDomain checks if the Domain is in a custom filter list.
+func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	if filterSubdomains {
+		// Check if domain is in the list and all its subdomains.
+		listOfDomains := splitDomain(fullDomain)
+		for _, domain := range listOfDomains {
+			_, ok := domainsFilterList[domain]
+			if ok {
+				return true, domain
+			}
+		}
+	} else {
+		// Check only if the domain is in the list
+		_, ok := domainsFilterList[fullDomain]
+		return ok, fullDomain
+	}
+	return false, ""
+}
+
+// LookupASN checks if the Autonomous system number is in a custom filter list.
+func LookupASN(number uint) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := autonomousSystemsFilterList[number]
+	return ok
+}
+
+// LookupCountry checks if the country code is in a custom filter list.
+func LookupCountry(countryCode string) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := countryCodesFilterList[countryCode]
+	return ok
+}
+
+func splitDomain(domain string) []string {
+	domain = strings.Trim(domain, ".")
+	suffix, _ := publicsuffix.PublicSuffix(domain)
+	if suffix == domain {
+		return []string{domain}
+	}
+
+	domainWithoutSuffix := domain[:len(domain)-len(suffix)]
+	domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".")
+
+	splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool {
+		return r == '.'
+	})
+
+	domains := make([]string, 0, len(splitted))
+	for idx := range splitted {
+
+		d := strings.Join(splitted[idx:], ".") + "." + suffix
+		if d[len(d)-1] != '.' {
+			d += "."
+		}
+		domains = append(domains, d)
+	}
+	return domains
+}

intel/module.go

@@ -2,11 +2,12 @@ package intel
 
 import (
 	"github.com/safing/portbase/modules"
+	_ "github.com/safing/portmaster/intel/customlists"
 )
 
 // Module of this package. Export needed for testing of the endpoints package.
 var Module *modules.Module
 
 func init() {
-	Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists")
+	Module = modules.Register("intel", nil, nil, nil, "geoip", "filterlists", "customlists")
 }

profile/config.go

@@ -64,9 +64,11 @@ var (
 	cfgOptionFilterLists      config.StringArrayOption
 	cfgOptionFilterListsOrder = 34
 
+	// Setting "Custom Filter List" at order 35.
+
 	CfgOptionFilterSubDomainsKey   = "filter/includeSubdomains"
 	cfgOptionFilterSubDomains      config.IntOption // security level option
-	cfgOptionFilterSubDomainsOrder = 35
+	cfgOptionFilterSubDomainsOrder = 36
 
 	// DNS Filtering.