wip: migrate to mono-repo. SPN has already been moved to spn/
This commit is contained in:
Patrick Pacher
55
service/intel/customlists/config.go
Normal file
55
service/intel/customlists/config.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package customlists
|
||||
|
||||
import (
|
||||
"github.com/safing/portbase/config"
|
||||
)
|
||||
|
||||
var (
|
||||
// CfgOptionCustomListFileKey is the config key for custom filter list file.
|
||||
CfgOptionCustomListFileKey = "filter/customListFile"
|
||||
cfgOptionCustomListFileOrder = 35
|
||||
cfgOptionCustomListCategoryAnnotation = "Filter Lists"
|
||||
)
|
||||
|
||||
var getFilePath config.StringOption
|
||||
|
||||
func registerConfig() error {
|
||||
help := `The file (.txt) is checked every couple minutes and will be automatically reloaded when it has changed.
|
||||
|
||||
Entries (one per line) may be one of:
|
||||
- Domain: "example.com"
|
||||
- IP Address: "10.0.0.1"
|
||||
- Country Code (based on IP): "US"
|
||||
- AS (Autonomous System): "AS1234"
|
||||
|
||||
Everything after the first element of a line, comments starting with a '#', and empty lines are ignored.
|
||||
The settings "Block Subdomains of Filter List Entries" and "Block Domain Aliases" also apply to the custom filter list.
|
||||
Lists in the "Hosts" format are not supported.
|
||||
|
||||
Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.`
|
||||
|
||||
// Register a setting for the file path in the ui
|
||||
err := config.Register(&config.Option{
|
||||
Name: "Custom Filter List",
|
||||
Key: CfgOptionCustomListFileKey,
|
||||
Description: "Specify the file path to a custom filter list (.txt), which will be automatically refreshed. Any connections matching a domain, IP address, Country or ASN in the file will be blocked.",
|
||||
Help: help,
|
||||
OptType: config.OptTypeString,
|
||||
ExpertiseLevel: config.ExpertiseLevelExpert,
|
||||
ReleaseLevel: config.ReleaseLevelStable,
|
||||
DefaultValue: "",
|
||||
RequiresRestart: false,
|
||||
Annotations: config.Annotations{
|
||||
config.DisplayOrderAnnotation: cfgOptionCustomListFileOrder,
|
||||
config.CategoryAnnotation: cfgOptionCustomListCategoryAnnotation,
|
||||
config.DisplayHintAnnotation: config.DisplayHintFilePicker,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
getFilePath = config.GetAsString(CfgOptionCustomListFileKey, "")
|
||||
|
||||
return nil
|
||||
}
|
||||
195
service/intel/customlists/lists.go
Normal file
195
service/intel/customlists/lists.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package customlists
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
|
||||
"github.com/safing/portbase/log"
|
||||
"github.com/safing/portbase/notifications"
|
||||
"github.com/safing/portmaster/service/network/netutils"
|
||||
)
|
||||
|
||||
var (
|
||||
countryCodesFilterList map[string]struct{}
|
||||
ipAddressesFilterList map[string]struct{}
|
||||
autonomousSystemsFilterList map[uint]struct{}
|
||||
domainsFilterList map[string]struct{}
|
||||
)
|
||||
|
||||
const (
|
||||
rationForInvalidLinesUntilWarning = 0.1
|
||||
parseStatusNotificationID = "customlists:parse-status"
|
||||
parseWarningNotificationID = "customlists:parse-warning"
|
||||
zeroIPNotificationID = "customlists:too-many-zero-ips"
|
||||
)
|
||||
|
||||
func initFilterLists() {
|
||||
countryCodesFilterList = make(map[string]struct{})
|
||||
ipAddressesFilterList = make(map[string]struct{})
|
||||
autonomousSystemsFilterList = make(map[uint]struct{})
|
||||
domainsFilterList = make(map[string]struct{})
|
||||
}
|
||||
|
||||
// IsLoaded returns whether a custom filter list is loaded.
|
||||
func IsLoaded() bool {
|
||||
filterListLock.RLock()
|
||||
defer filterListLock.RUnlock()
|
||||
|
||||
switch {
|
||||
case len(domainsFilterList) > 0:
|
||||
return true
|
||||
case len(ipAddressesFilterList) > 0:
|
||||
return true
|
||||
case len(countryCodesFilterList) > 0:
|
||||
return true
|
||||
case len(autonomousSystemsFilterList) > 0:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func parseFile(filePath string) error {
|
||||
// Reset all maps, previous (if any) settings will be lost.
|
||||
for key := range countryCodesFilterList {
|
||||
delete(countryCodesFilterList, key)
|
||||
}
|
||||
for key := range ipAddressesFilterList {
|
||||
delete(ipAddressesFilterList, key)
|
||||
}
|
||||
for key := range autonomousSystemsFilterList {
|
||||
delete(autonomousSystemsFilterList, key)
|
||||
}
|
||||
for key := range domainsFilterList {
|
||||
delete(domainsFilterList, key)
|
||||
}
|
||||
|
||||
// Ignore empty file path.
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open the file if possible
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
log.Warningf("intel/customlists: failed to parse file %s", err)
|
||||
module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error())
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
var allLinesCount uint64
|
||||
var invalidLinesCount uint64
|
||||
|
||||
// Read filter file line by line.
|
||||
scanner := bufio.NewScanner(file)
|
||||
// The scanner will error out if the line is greater than 64K, in this case it is enough.
|
||||
for scanner.Scan() {
|
||||
allLinesCount++
|
||||
// Parse and count invalid lines (comment, empty lines, zero IPs...)
|
||||
if !parseLine(scanner.Text()) {
|
||||
invalidLinesCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check for scanner error.
|
||||
if err := scanner.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount)
|
||||
|
||||
if invalidLinesRation > rationForInvalidLinesUntilWarning {
|
||||
log.Warning("intel/customlists: Too many invalid lines")
|
||||
module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines",
|
||||
fmt.Sprintf(`%d out of %d lines are invalid.
|
||||
Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount))
|
||||
} else {
|
||||
module.Resolve(zeroIPNotificationID)
|
||||
}
|
||||
|
||||
allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList)
|
||||
log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath)
|
||||
|
||||
notifications.NotifyInfo(parseStatusNotificationID,
|
||||
"Custom filter list loaded successfully.",
|
||||
fmt.Sprintf(`Custom filter list loaded from file %s:
|
||||
%d Domains
|
||||
%d IPs
|
||||
%d Autonomous Systems
|
||||
%d Countries`,
|
||||
filePath,
|
||||
len(domainsFilterList),
|
||||
len(ipAddressesFilterList),
|
||||
len(autonomousSystemsFilterList),
|
||||
len(countryCodesFilterList)))
|
||||
|
||||
module.Resolve(parseWarningNotificationID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseLine(line string) (valid bool) {
|
||||
// Everything after the first field will be ignored.
|
||||
fields := strings.Fields(line)
|
||||
|
||||
// Ignore empty lines.
|
||||
if len(fields) == 0 {
|
||||
return true // Not an entry, but a valid line.
|
||||
}
|
||||
|
||||
field := fields[0]
|
||||
|
||||
// Ignore comments
|
||||
if strings.HasPrefix(field, "#") {
|
||||
return true // Not an entry, but a valid line.
|
||||
}
|
||||
|
||||
// Go through all possible field types.
|
||||
// Parsing is ordered by
|
||||
// 1. Parsing options (ie. the domain has most variation and goes last.)
|
||||
// 2. Speed
|
||||
|
||||
// Check if it'a a country code.
|
||||
if isCountryCode(field) {
|
||||
countryCodesFilterList[field] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if it's a Autonomous system (example AS123).
|
||||
if isAutonomousSystem(field) {
|
||||
asNumber, err := strconv.ParseUint(field[2:], 10, 32)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
autonomousSystemsFilterList[uint(asNumber)] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
// Try to parse IP address.
|
||||
ip := net.ParseIP(field)
|
||||
if ip != nil {
|
||||
// Check for zero ip.
|
||||
if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) {
|
||||
return false
|
||||
}
|
||||
|
||||
ipAddressesFilterList[ip.String()] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if it's a domain.
|
||||
domain := dns.Fqdn(field)
|
||||
if netutils.IsValidFqdn(domain) {
|
||||
domainsFilterList[domain] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
207
service/intel/customlists/module.go
Normal file
207
service/intel/customlists/module.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package customlists
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/publicsuffix"
|
||||
|
||||
"github.com/safing/portbase/api"
|
||||
"github.com/safing/portbase/modules"
|
||||
)
|
||||
|
||||
var module *modules.Module
|
||||
|
||||
const (
|
||||
configModuleName = "config"
|
||||
configChangeEvent = "config change"
|
||||
)
|
||||
|
||||
// Helper variables for parsing the input file.
|
||||
var (
|
||||
isCountryCode = regexp.MustCompile("^[A-Z]{2}$").MatchString
|
||||
isAutonomousSystem = regexp.MustCompile(`^AS[0-9]+$`).MatchString
|
||||
)
|
||||
|
||||
var (
|
||||
filterListFilePath string
|
||||
filterListFileModifiedTime time.Time
|
||||
|
||||
filterListLock sync.RWMutex
|
||||
parserTask *modules.Task
|
||||
|
||||
// ErrNotConfigured is returned when updating the custom filter list, but it
|
||||
// is not configured.
|
||||
ErrNotConfigured = errors.New("custom filter list not configured")
|
||||
)
|
||||
|
||||
func init() {
|
||||
module = modules.Register("customlists", prep, start, nil, "base")
|
||||
}
|
||||
|
||||
func prep() error {
|
||||
initFilterLists()
|
||||
|
||||
// Register the config in the ui.
|
||||
err := registerConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Register api endpoint for updating the filter list.
|
||||
if err := api.RegisterEndpoint(api.Endpoint{
|
||||
Path: "customlists/update",
|
||||
Write: api.PermitUser,
|
||||
BelongsTo: module,
|
||||
ActionFunc: func(ar *api.Request) (msg string, err error) {
|
||||
errCheck := checkAndUpdateFilterList()
|
||||
if errCheck != nil {
|
||||
return "", errCheck
|
||||
}
|
||||
return "Custom filter list loaded successfully.", nil
|
||||
},
|
||||
Name: "Update custom filter list",
|
||||
Description: "Reload the filter list from the configured file.",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func start() error {
|
||||
// Register to hook to update after config change.
|
||||
if err := module.RegisterEventHook(
|
||||
configModuleName,
|
||||
configChangeEvent,
|
||||
"update custom filter list",
|
||||
func(ctx context.Context, obj interface{}) error {
|
||||
if err := checkAndUpdateFilterList(); !errors.Is(err, ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
|
||||
parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error {
|
||||
_ = checkAndUpdateFilterList()
|
||||
return nil
|
||||
}).Schedule(time.Now().Add(20 * time.Second))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkAndUpdateFilterList() error {
|
||||
filterListLock.Lock()
|
||||
defer filterListLock.Unlock()
|
||||
|
||||
// Get path and return error if empty
|
||||
filePath := getFilePath()
|
||||
if filePath == "" {
|
||||
return ErrNotConfigured
|
||||
}
|
||||
|
||||
// Schedule next update check
|
||||
parserTask.Schedule(time.Now().Add(1 * time.Minute))
|
||||
|
||||
// Try to get file info
|
||||
modifiedTime := time.Now()
|
||||
if fileInfo, err := os.Stat(filePath); err == nil {
|
||||
modifiedTime = fileInfo.ModTime()
|
||||
}
|
||||
|
||||
// Check if file path has changed or if modified time has changed
|
||||
if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) {
|
||||
err := parseFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filterListFileModifiedTime = modifiedTime
|
||||
filterListFilePath = filePath
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LookupIP checks if the IP address is in a custom filter list.
|
||||
func LookupIP(ip net.IP) bool {
|
||||
filterListLock.RLock()
|
||||
defer filterListLock.RUnlock()
|
||||
|
||||
_, ok := ipAddressesFilterList[ip.String()]
|
||||
return ok
|
||||
}
|
||||
|
||||
// LookupDomain checks if the Domain is in a custom filter list.
|
||||
func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
|
||||
filterListLock.RLock()
|
||||
defer filterListLock.RUnlock()
|
||||
|
||||
if filterSubdomains {
|
||||
// Check if domain is in the list and all its subdomains.
|
||||
listOfDomains := splitDomain(fullDomain)
|
||||
for _, domain := range listOfDomains {
|
||||
_, ok := domainsFilterList[domain]
|
||||
if ok {
|
||||
return true, domain
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Check only if the domain is in the list
|
||||
_, ok := domainsFilterList[fullDomain]
|
||||
return ok, fullDomain
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// LookupASN checks if the Autonomous system number is in a custom filter list.
|
||||
func LookupASN(number uint) bool {
|
||||
filterListLock.RLock()
|
||||
defer filterListLock.RUnlock()
|
||||
|
||||
_, ok := autonomousSystemsFilterList[number]
|
||||
return ok
|
||||
}
|
||||
|
||||
// LookupCountry checks if the country code is in a custom filter list.
|
||||
func LookupCountry(countryCode string) bool {
|
||||
filterListLock.RLock()
|
||||
defer filterListLock.RUnlock()
|
||||
|
||||
_, ok := countryCodesFilterList[countryCode]
|
||||
return ok
|
||||
}
|
||||
|
||||
func splitDomain(domain string) []string {
|
||||
domain = strings.Trim(domain, ".")
|
||||
suffix, _ := publicsuffix.PublicSuffix(domain)
|
||||
if suffix == domain {
|
||||
return []string{domain}
|
||||
}
|
||||
|
||||
domainWithoutSuffix := domain[:len(domain)-len(suffix)]
|
||||
domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".")
|
||||
|
||||
splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool {
|
||||
return r == '.'
|
||||
})
|
||||
|
||||
domains := make([]string, 0, len(splitted))
|
||||
for idx := range splitted {
|
||||
|
||||
d := strings.Join(splitted[idx:], ".") + "." + suffix
|
||||
if d[len(d)-1] != '.' {
|
||||
d += "."
|
||||
}
|
||||
domains = append(domains, d)
|
||||
}
|
||||
return domains
|
||||
}
|
||||
Reference in New Issue
Block a user