Merge branch 'master' into merge/master-into-v2-with-history (v1 into v2)

# Conflicts:
#	Earthfile
#	base/config/main.go
#	base/database/dbmodule/db.go
#	base/database/main.go
#	go.mod
#	go.sum
#	service/core/base/global.go

service/core/base/databases.go

@@ -1,43 +1,51 @@
 package base
 
 import (
+	"path/filepath"
+
 	"github.com/safing/portmaster/base/database"
-	_ "github.com/safing/portmaster/base/database/dbmodule"
 	_ "github.com/safing/portmaster/base/database/storage/bbolt"
+	_ "github.com/safing/portmaster/base/database/storage/sqlite"
+	"github.com/safing/portmaster/base/utils"
 )
 
 // Default Values (changeable for testing).
 var (
-	DefaultDatabaseStorageType = "bbolt"
+	DefaultDatabaseStorageType = "sqlite"
 )
 
 func registerDatabases() error {
+	// If there is an existing bbolt core database, use it instead.
+	coreStorageType := DefaultDatabaseStorageType
+	if utils.PathExists(filepath.Join(module.instance.DataDir(), "databases", "core", "bbolt")) {
+		coreStorageType = "bbolt"
+	}
+
+	// Register core database.
 	_, err := database.Register(&database.Database{
 		Name:        "core",
 		Description: "Holds core data, such as settings and profiles",
-		StorageType: DefaultDatabaseStorageType,
+		StorageType: coreStorageType,
 	})
 	if err != nil {
 		return err
 	}
 
+	// If there is an existing cache bbolt database, use it instead.
+	cacheStorageType := DefaultDatabaseStorageType
+	if utils.PathExists(filepath.Join(module.instance.DataDir(), "databases", "cache", "bbolt")) {
+		cacheStorageType = "bbolt"
+	}
+
+	// Register cache database.
 	_, err = database.Register(&database.Database{
 		Name:        "cache",
 		Description: "Cached data, such as Intelligence and DNS Records",
-		StorageType: DefaultDatabaseStorageType,
+		StorageType: cacheStorageType,
 	})
 	if err != nil {
 		return err
 	}
 
-	// _, err = database.Register(&database.Database{
-	//   Name:        "history",
-	//   Description: "Historic event data",
-	//   StorageType: DefaultDatabaseStorageType,
-	// })
-	// if err != nil {
-	//   return err
-	// }
-
 	return nil
 }

service/firewall/interception/windowskext2/handler.go

@@ -107,6 +107,7 @@ func Handler(ctx context.Context, packets chan packet.Packet, bandwidthUpdate ch
 				newPacket := &Packet{
 					verdictRequest: conn.ID,
 					payload:        conn.Payload,
+					payloadLayer:   conn.PayloadLayer,
 					verdictSet:     abool.NewBool(false),
 				}
 				info := newPacket.Info()

service/intel/entity.go

@@ -397,28 +397,32 @@ func (e *Entity) getDomainLists(ctx context.Context) {
 }
 
 func splitDomain(domain string) []string {
-	domain = strings.Trim(domain, ".")
-	suffix, _ := publicsuffix.PublicSuffix(domain)
-	if suffix == domain {
+	// Get suffix.
+	d := strings.TrimSuffix(domain, ".")
+	suffix, icann := publicsuffix.PublicSuffix(d)
+	if suffix == d {
 		return []string{domain}
 	}
 
-	domainWithoutSuffix := domain[:len(domain)-len(suffix)]
-	domainWithoutSuffix = strings.Trim(domainWithoutSuffix, ".")
-
-	splitted := strings.FieldsFunc(domainWithoutSuffix, func(r rune) bool {
+	// Split all subdomain into labels.
+	labels := strings.FieldsFunc(d[:len(d)-len(suffix)], func(r rune) bool {
 		return r == '.'
 	})
 
-	domains := make([]string, 0, len(splitted))
-	for idx := range splitted {
-
-		d := strings.Join(splitted[idx:], ".") + "." + suffix
-		if d[len(d)-1] != '.' {
-			d += "."
-		}
-		domains = append(domains, d)
+	// Build list of all domains up to the public suffix.
+	domains := make([]string, 0, len(labels)+1)
+	for idx := range labels {
+		domains = append(
+			domains,
+			strings.Join(labels[idx:], ".")+"."+suffix+".",
+		)
 	}
+
+	// If the suffix is not a real TLD, but a public suffix, add it to the list.
+	if !icann {
+		domains = append(domains, suffix+".")
+	}
+
 	return domains
 }
 

service/intel/entity_test.go

@@ -0,0 +1,33 @@
+package intel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var splitDomainTestCases = [][]string{
+	// Regular registered domains and subdomains.
+	{"example.com."},
+	{"www.example.com.", "example.com."},
+	{"sub.domain.example.com.", "domain.example.com.", "example.com."},
+	{"example.co.uk."},
+	{"www.example.co.uk.", "example.co.uk."},
+
+	// TLD or public suffix: Return as is.
+	{"com."},
+	{"googleapis.com."},
+
+	// Public suffix domains: Return including public suffix.
+	{"test.googleapis.com.", "googleapis.com."},
+	{"sub.domain.googleapis.com.", "domain.googleapis.com.", "googleapis.com."},
+}
+
+func TestSplitDomain(t *testing.T) {
+	t.Parallel()
+
+	for _, testCase := range splitDomainTestCases {
+		splitted := splitDomain(testCase[0])
+		assert.Equal(t, testCase, splitted, "result must match")
+	}
+}

service/intel/filterlists/database.go

@@ -144,7 +144,7 @@ func persistRecords(startJob func(func() error), records <-chan record.Record) {
 
 		timePerEntity := time.Since(start) / time.Duration(cnt)
 		speed := float64(time.Second) / float64(timePerEntity)
-		log.Debugf("processed %d entities in %s with %s / entity (%.2f entities/second)", cnt, time.Since(start), timePerEntity, speed)
+		log.Debugf("intel/filterlists: processed %d entities in %s with %s / entity (%.2f entities/second)", cnt, time.Since(start), timePerEntity, speed)
 	}
 
 	batch := database.NewInterface(&database.Options{Local: true, Internal: true})

service/intel/filterlists/updater.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/safing/portmaster/base/database"
 	"github.com/safing/portmaster/base/database/query"
+	"github.com/safing/portmaster/base/database/storage"
 	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/updates"
@@ -158,9 +159,25 @@ func performUpdate(ctx context.Context) error {
 
 func removeAllObsoleteFilterEntries(wc *mgr.WorkerCtx) error {
 	log.Debugf("intel/filterlists: cleanup task started, removing obsolete filter list entries ...")
-	n, err := cache.Purge(wc.Ctx(), query.New(filterListKeyPrefix).Where(
-		// TODO(ppacher): remember the timestamp we started the last update
-		// and use that rather than "one hour ago"
+
+	// TODO: Remember the timestamp we started the last update and use that rather than "one hour ago".
+
+	// First try to purge with PurgeOlderThan.
+	n, err := cache.PurgeOlderThan(wc.Ctx(), filterListKeyPrefix, time.Now().Add(-time.Hour))
+	switch {
+	case err == nil:
+		// Success!
+		log.Debugf("intel/filterlists: successfully removed %d obsolete entries", n)
+		return nil
+	case errors.Is(err, database.ErrNotImplemented) || errors.Is(err, storage.ErrNotImplemented):
+		// Try next method.
+	default:
+		// Return error.
+		return err
+	}
+
+	// Try with regular purge.
+	n, err = cache.Purge(wc.Ctx(), query.New(filterListKeyPrefix).Where(
 		query.Where("UpdatedAt", query.LessThan, time.Now().Add(-time.Hour).Unix()),
 	))
 	if err != nil {