wip: migrate to mono-repo. SPN has already been moved to spn/

service/intel/customlists/module.go

@@ -0,0 +1,207 @@
+package customlists
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/publicsuffix"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/modules"
+)
+
+var module *modules.Module
+
+const (
+	configModuleName  = "config"
+	configChangeEvent = "config change"
+)
+
+// Helper variables for parsing the input file.
+var (
+	isCountryCode      = regexp.MustCompile("^[A-Z]{2}$").MatchString
+	isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString
+)
+
+var (
+	filterListFilePath         string
+	filterListFileModifiedTime time.Time
+
+	filterListLock sync.RWMutex
+	parserTask     *modules.Task
+
+	// ErrNotConfigured is returned when updating the custom filter list, but it
+	// is not configured.
+	ErrNotConfigured = errors.New("custom filter list not configured")
+)
+
+func init() {
+	module = modules.Register("customlists", prep, start, nil, "base")
+}
+
+func prep() error {
+	initFilterLists()
+
+	// Register the config in the ui.
+	err := registerConfig()
+	if err != nil {
+		return err
+	}
+
+	// Register api endpoint for updating the filter list.
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:      "customlists/update",
+		Write:     api.PermitUser,
+		BelongsTo: module,
+		ActionFunc: func(ar *api.Request) (msg string, err error) {
+			errCheck := checkAndUpdateFilterList()
+			if errCheck != nil {
+				return "", errCheck
+			}
+			return "Custom filter list loaded successfully.", nil
+		},
+		Name:        "Update custom filter list",
+		Description: "Reload the filter list from the configured file.",
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func start() error {
+	// Register to hook to update after config change.
+	if err := module.RegisterEventHook(
+		configModuleName,
+		configChangeEvent,
+		"update custom filter list",
+		func(ctx context.Context, obj interface{}) error {
+			if err := checkAndUpdateFilterList(); !errors.Is(err, ErrNotConfigured) {
+				return err
+			}
+			return nil
+		},
+	); err != nil {
+		return err
+	}
+
+	// Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
+	parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error {
+		_ = checkAndUpdateFilterList()
+		return nil
+	}).Schedule(time.Now().Add(20 * time.Second))
+
+	return nil
+}
+
+func checkAndUpdateFilterList() error {
+	filterListLock.Lock()
+	defer filterListLock.Unlock()
+
+	// Get path and return error if empty
+	filePath := getFilePath()
+	if filePath == "" {
+		return ErrNotConfigured
+	}
+
+	// Schedule next update check
+	parserTask.Schedule(time.Now().Add(1 * time.Minute))
+
+	// Try to get file info
+	modifiedTime := time.Now()
+	if fileInfo, err := os.Stat(filePath); err == nil {
+		modifiedTime = fileInfo.ModTime()
+	}
+
+	// Check if file path has changed or if modified time has changed
+	if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) {
+		err := parseFile(filePath)
+		if err != nil {
+			return err
+		}
+		filterListFileModifiedTime = modifiedTime
+		filterListFilePath = filePath
+	}
+	return nil
+}
+
+// LookupIP checks if the IP address is in a custom filter list.
+func LookupIP(ip net.IP) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := ipAddressesFilterList[ip.String()]
+	return ok
+}
+
+// LookupDomain checks if the Domain is in a custom filter list.
+func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	if filterSubdomains {
+		// Check if domain is in the list and all its subdomains.
+		listOfDomains := splitDomain(fullDomain)
+		for _, domain := range listOfDomains {
+			_, ok := domainsFilterList[domain]
+			if ok {
+				return true, domain
+			}
+		}
+	} else {
+		// Check only if the domain is in the list
+		_, ok := domainsFilterList[fullDomain]
+		return ok, fullDomain
+	}
+	return false, ""
+}
+
+// LookupASN checks if the Autonomous system number is in a custom filter list.
+func LookupASN(number uint) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := autonomousSystemsFilterList[number]
+	return ok
+}
+
+// LookupCountry checks if the country code is in a custom filter list.
+func LookupCountry(countryCode string) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
+	_, ok := countryCodesFilterList[countryCode]
+	return ok
+}
+
+func splitDomain(domain string) []string {
+	domain = strings.Trim(domain, ".")
+	suffix, _ := publicsuffix.PublicSuffix(domain)
+	if suffix == domain {
+		return []string{domain}
+	}
+
+	domainWithoutSuffix := domain[:len(domain)-len(suffix)]
+	domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".")
+
+	splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool {
+		return r == '.'
+	})
+
+	domains := make([]string, 0, len(splitted))
+	for idx := range splitted {
+
+		d := strings.Join(splitted[idx:], ".") + "." + suffix
+		if d[len(d)-1] != '.' {
+			d += "."
+		}
+		domains = append(domains, d)
+	}
+	return domains
+}