From a92410fe27bb094cc8021bb332c155c8ca40636e Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:21:44 +0200
Subject: [PATCH 01/86] Clear DNS cache when DNS resolver config changes in any
 way

---
 resolver/api.go        |  2 +-
 resolver/namerecord.go | 12 +++++++++---
 resolver/resolvers.go  | 28 +++++++++++++++++++++-------
 3 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/resolver/api.go b/resolver/api.go
index a31b19a1..72703462 100644
--- a/resolver/api.go
+++ b/resolver/api.go
@@ -12,7 +12,7 @@ func registerAPI() error {
 		Path:        "dns/clear",
 		Write:       api.PermitUser,
 		BelongsTo:   module,
-		ActionFunc:  clearNameCache,
+		ActionFunc:  clearNameCacheHandler,
 		Name:        "Clear cached DNS records",
 		Description: "Deletes all saved DNS records from the database.",
 	}); err != nil {
diff --git a/resolver/namerecord.go b/resolver/namerecord.go
index 8e4841f2..e3db05e0 100644
--- a/resolver/namerecord.go
+++ b/resolver/namerecord.go
@@ -1,6 +1,7 @@
 package resolver
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
@@ -129,13 +130,18 @@ func (nameRecord *NameRecord) Save() error {
 	return recordDatabase.PutNew(nameRecord)
 }
 
-// clearNameCache clears all dns caches from the database.
-func clearNameCache(ar *api.Request) (msg string, err error) {
+// clearNameCacheHandler is an API handler that clears all dns caches from the database.
+func clearNameCacheHandler(ar *api.Request) (msg string, err error) {
 	log.Info("resolver: user requested dns cache clearing via action")
 
+	return clearNameCache(ar.Context())
+}
+
+// clearNameCache clears all dns caches from the database.
+func clearNameCache(ctx context.Context) (msg string, err error) {
 	recordDatabase.FlushCache()
 	recordDatabase.ClearCache()
-	n, err := recordDatabase.Purge(ar.Context(), query.New(nameRecordsKeyPrefix))
+	n, err := recordDatabase.Purge(ctx, query.New(nameRecordsKeyPrefix))
 	if err != nil {
 		return "", err
 	}
diff --git a/resolver/resolvers.go b/resolver/resolvers.go
index 0f7eaa16..4360d836 100644
--- a/resolver/resolvers.go
+++ b/resolver/resolvers.go
@@ -1,6 +1,7 @@
 package resolver
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -34,12 +35,13 @@ const (
 )
 
 var (
-	globalResolvers []*Resolver          // all (global) resolvers
-	localResolvers  []*Resolver          // all resolvers that are in site-local or link-local IP ranges
-	systemResolvers []*Resolver          // all resolvers that were assigned by the system
-	localScopes     []*Scope             // list of scopes with a list of local resolvers that can resolve the scope
-	activeResolvers map[string]*Resolver // lookup map of all resolvers
-	resolversLock   sync.RWMutex
+	globalResolvers       []*Resolver          // all (global) resolvers
+	localResolvers        []*Resolver          // all resolvers that are in site-local or link-local IP ranges
+	systemResolvers       []*Resolver          // all resolvers that were assigned by the system
+	localScopes           []*Scope             // list of scopes with a list of local resolvers that can resolve the scope
+	activeResolvers       map[string]*Resolver // lookup map of all resolvers
+	currentResolverConfig []string             // current active resolver config, to detect changes
+	resolversLock         sync.RWMutex
 )
 
 func indexOfScope(domain string, list []*Scope) int {
@@ -285,8 +287,20 @@ func loadResolvers() {
 	// Resolve module error about missing resolvers.
 	module.Resolve(missingResolversErrorID)
 
+	// Check if settings were changed and clear name cache when they did.
+	newResolverConfig := configuredNameServers()
+	if len(currentResolverConfig) > 0 &&
+		!utils.StringSliceEqual(currentResolverConfig, newResolverConfig) {
+		module.StartWorker("clear dns cache", func(ctx context.Context) error {
+			log.Info("resolver: clearing dns cache due to changed resolver config")
+			_, err := clearNameCache(ctx)
+			return err
+		})
+	}
+	currentResolverConfig = newResolverConfig
+
 	newResolvers := append(
-		getConfiguredResolvers(configuredNameServers()),
+		getConfiguredResolvers(newResolverConfig),
 		getSystemResolvers()...,
 	)
 

From 801fd33a80f32ef2dfddb25c63c0dd01dc5abec7 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:22:17 +0200
Subject: [PATCH 02/86] Improve resolver config for intro slides

---
 resolver/config.go | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/resolver/config.go b/resolver/config.go
index b274f119..8e9e8109 100644
--- a/resolver/config.go
+++ b/resolver/config.go
@@ -31,7 +31,7 @@ var (
 
 		// Cloudflare (encrypted DNS, with malware protection)
 		`dot://1.1.1.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip`,
-		// `dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip`,
+		`dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip`,
 
 		// AdGuard (encrypted DNS, default flavor)
 		// `dot://94.140.14.14:853?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip`,
@@ -107,6 +107,7 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 	- "search": specify prioritized domains/TLDs for this resolver (delimited by ",")
 	- "search-only": use this resolver for domains in the "search" parameter only (no value)
 `, `"`, "`"),
+		Sensitive:       true,
 		OptType:         config.OptTypeStringArray,
 		ExpertiseLevel:  config.ExpertiseLevelExpert,
 		ReleaseLevel:    config.ReleaseLevelStable,
@@ -118,6 +119,14 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 			config.DisplayOrderAnnotation: cfgOptionNameServersOrder,
 			config.CategoryAnnotation:     "Servers",
 			config.QuickSettingsAnnotation: []config.QuickSetting{
+				{
+					Name:   "Cloudflare (with Malware Filter)",
+					Action: config.QuickReplace,
+					Value: []string{
+						"dot://1.1.1.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+					},
+				},
 				{
 					Name:   "Quad9",
 					Action: config.QuickReplace,
@@ -142,14 +151,6 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 						"dot://94.130.106.88:443?verify=dot1.applied-privacy.net&name=AppliedPrivacy",
 					},
 				},
-				{
-					Name:   "Cloudflare (with Malware Filter)",
-					Action: config.QuickReplace,
-					Value: []string{
-						"dot://1.1.1.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
-						"dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
-					},
-				},
 			},
 			"self:detail:internalSpecialUseDomains": internalSpecialUseDomains,
 			"self:detail:connectivityDomains":       netenv.ConnectivityDomains,

From 17f67be52168a836983700c49ca0e5994192c39f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:25:33 +0200
Subject: [PATCH 03/86] Increase self-check timeouts and threshold

---
 compat/module.go    | 6 +++++-
 compat/selfcheck.go | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/compat/module.go b/compat/module.go
index 2cc6eca7..c87afd1a 100644
--- a/compat/module.go
+++ b/compat/module.go
@@ -28,6 +28,10 @@ var (
 	selfcheckFails int
 )
 
+// selfcheckFailThreshold holds the threshold of how many times the selfcheck
+// must fail before it is reported.
+const selfcheckFailThreshold = 5
+
 func init() {
 	module = modules.Register("compat", prep, start, stop, "base", "network", "interception", "netenv", "notifications")
 
@@ -82,7 +86,7 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 		selfcheckFails++
 
 		log.Errorf("compat: %s", err)
-		if selfcheckFails >= 3 {
+		if selfcheckFails >= selfcheckFailThreshold {
 			issue.notify(err)
 		}
 
diff --git a/compat/selfcheck.go b/compat/selfcheck.go
index 4aee02a7..1931b70e 100644
--- a/compat/selfcheck.go
+++ b/compat/selfcheck.go
@@ -28,12 +28,12 @@ var (
 	systemIntegrationCheckDialNet      = fmt.Sprintf("ip4:%d", uint8(SystemIntegrationCheckProtocol))
 	systemIntegrationCheckDialIP       = SystemIntegrationCheckDstIP.String()
 	systemIntegrationCheckPackets      = make(chan packet.Packet, 1)
-	systemIntegrationCheckWaitDuration = 10 * time.Second
+	systemIntegrationCheckWaitDuration = 30 * time.Second
 
 	// DNSCheckInternalDomainScope is the domain scope to use for dns checks.
 	DNSCheckInternalDomainScope = ".self-check." + resolver.InternalSpecialUseDomain
 	dnsCheckReceivedDomain      = make(chan string, 1)
-	dnsCheckWaitDuration        = 10 * time.Second
+	dnsCheckWaitDuration        = 30 * time.Second
 	dnsCheckAnswerLock          sync.Mutex
 	dnsCheckAnswer              net.IP
 )

From bff3dc8a274f614fd6902cb4d3f1e2747c7149b7 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:27:08 +0200
Subject: [PATCH 04/86] Fix secure DNS bypass notification to reflect reality

---
 compat/notify.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compat/notify.go b/compat/notify.go
index cc317d21..893f60d4 100644
--- a/compat/notify.go
+++ b/compat/notify.go
@@ -43,9 +43,9 @@ var (
 
 	secureDNSBypassIssue = &appIssue{
 		id:    "compat:secure-dns-bypass-%s",
-		title: "Detected %s Bypass Attempt",
-		message: `[APPNAME] is bypassing Portmaster's firewall functions through its Secure DNS resolver. Portmaster can no longer protect or filter connections coming from [APPNAME]. Disable Secure DNS within [APPNAME] to restore functionality.  
-Rest assured that Portmaster already handles Secure DNS for your whole device.`,
+		title: "Prevented %s Bypass Attempt",
+		message: `[APPNAME] attempted to bypass Portmaster's firewall functions by directly using a Secure DNS resolver. If [APPNAME] experiences problems, disable Secure DNS within [APPNAME] to restore functionality. Rest assured that Portmaster already handles Secure DNS for your whole device, including [APPNAME].  
+If bypass prevention for [APPNAME] is disabled in Portmaster, it can no longer protect or filter connections coming from [APPNAME].`,
 		// TODO: Add this when the new docs page is finished:
 		// , or [find out about other options](link to new docs page)
 		level: notifications.Warning,

From 7d1f7a0d0f1635dfb4f320eb6a1951d3dc36c5a1 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:27:49 +0200
Subject: [PATCH 05/86] Add notify threshold for app related notification

---
 compat/module.go |  3 +++
 compat/notify.go | 65 +++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/compat/module.go b/compat/module.go
index c87afd1a..1517f2cc 100644
--- a/compat/module.go
+++ b/compat/module.go
@@ -52,6 +52,9 @@ func start() error {
 		MaxDelay(selfcheckTaskRetryAfter).
 		Schedule(time.Now().Add(selfcheckTaskRetryAfter))
 
+	module.NewTask("clean notify thresholds", cleanNotifyThreshold).
+		Repeat(10 * time.Minute)
+
 	return module.RegisterEventHook(
 		netenv.ModuleName,
 		netenv.NetworkChangedEvent,
diff --git a/compat/notify.go b/compat/notify.go
index 893f60d4..9a263e7a 100644
--- a/compat/notify.go
+++ b/compat/notify.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/notifications"
 	"github.com/safing/portmaster/process"
 	"github.com/safing/portmaster/profile"
@@ -124,9 +125,6 @@ func (issue *appIssue) notify(proc *process.Process) {
 		proc.Path,
 	)
 
-	// Build message.
-	message := strings.ReplaceAll(issue.message, "[APPNAME]", p.Name)
-
 	// Check if we already have this notification.
 	eventID := fmt.Sprintf(issue.id, p.ID)
 	n := notifications.Get(eventID)
@@ -134,7 +132,15 @@ func (issue *appIssue) notify(proc *process.Process) {
 		return
 	}
 
-	// Otherwise, create a new one.
+	// Check if we reach the threshold to actually send a notification.
+	if !isOverThreshold(eventID) {
+		return
+	}
+
+	// Build message.
+	message := strings.ReplaceAll(issue.message, "[APPNAME]", p.Name)
+
+	// Create a new notification.
 	n = &notifications.Notification{
 		EventID:      eventID,
 		Type:         issue.level,
@@ -171,3 +177,54 @@ func (issue *appIssue) notify(proc *process.Process) {
 		return nil
 	})
 }
+
+const (
+	notifyThresholdMinIncidents = 11
+	notifyThresholdResetAfter   = 2 * time.Minute
+)
+
+var (
+	notifyThresholds     = make(map[string]*notifyThreshold)
+	notifyThresholdsLock sync.Mutex
+)
+
+type notifyThreshold struct {
+	FirstSeen time.Time
+	Incidents uint
+}
+
+func (nt *notifyThreshold) expired() bool {
+	return time.Now().Add(-notifyThresholdResetAfter).After(nt.FirstSeen)
+}
+
+func isOverThreshold(id string) bool {
+	notifyThresholdsLock.Lock()
+	defer notifyThresholdsLock.Unlock()
+
+	// Get notify threshold and check if we reach the minimum incidents.
+	nt, ok := notifyThresholds[id]
+	if ok && !nt.expired() {
+		nt.Incidents++
+		return nt.Incidents >= notifyThresholdMinIncidents
+	}
+
+	// Add new entry.
+	notifyThresholds[id] = &notifyThreshold{
+		FirstSeen: time.Now(),
+		Incidents: 1,
+	}
+	return false
+}
+
+func cleanNotifyThreshold(ctx context.Context, task *modules.Task) error {
+	notifyThresholdsLock.Lock()
+	defer notifyThresholdsLock.Unlock()
+
+	for id, nt := range notifyThresholds {
+		if nt.expired() {
+			delete(notifyThresholds, id)
+		}
+	}
+
+	return nil
+}

From ec09c8a9485c321cbe006e0e838a156d6ddd7786 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:28:13 +0200
Subject: [PATCH 06/86] Fix connection locking in firewall packet handler

---
 network/connection.go | 64 ++++++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/network/connection.go b/network/connection.go
index 31219259..2756cffa 100644
--- a/network/connection.go
+++ b/network/connection.go
@@ -580,10 +580,7 @@ func (conn *Connection) SetFirewallHandler(handler FirewallHandler) {
 		conn.pktQueue = make(chan packet.Packet, 1000)
 
 		// start handling
-		module.StartWorker("packet handler", func(ctx context.Context) error {
-			conn.packetHandler()
-			return nil
-		})
+		module.StartWorker("packet handler", conn.packetHandlerWorker)
 	}
 	conn.firewallHandler = handler
 }
@@ -608,35 +605,46 @@ func (conn *Connection) HandlePacket(pkt packet.Packet) {
 	}
 }
 
-// packetHandler sequentially handles queued packets.
-func (conn *Connection) packetHandler() {
-	for pkt := range conn.pktQueue {
-		if pkt == nil {
-			return
+// packetHandlerWorker sequentially handles queued packets.
+func (conn *Connection) packetHandlerWorker(ctx context.Context) error {
+	for {
+		select {
+		case pkt := <-conn.pktQueue:
+			if pkt == nil {
+				return nil
+			}
+			packetHandlerHandleConn(conn, pkt)
+
+		case <-ctx.Done():
+			conn.Lock()
+			defer conn.Unlock()
+			conn.firewallHandler = nil
+			return nil
 		}
-		// get handler
-		conn.Lock()
+	}
+}
 
-		// execute handler or verdict
-		if conn.firewallHandler != nil {
-			conn.firewallHandler(conn, pkt)
-		} else {
-			defaultFirewallHandler(conn, pkt)
-		}
+func packetHandlerHandleConn(conn *Connection, pkt packet.Packet) {
+	conn.Lock()
+	defer conn.Unlock()
 
-		// log verdict
-		log.Tracer(pkt.Ctx()).Infof("filter: connection %s %s: %s", conn, conn.Verdict.Verb(), conn.Reason.Msg)
-		// submit trace logs
-		log.Tracer(pkt.Ctx()).Submit()
+	// Handle packet with appropriate handler.
+	if conn.firewallHandler != nil {
+		conn.firewallHandler(conn, pkt)
+	} else {
+		defaultFirewallHandler(conn, pkt)
+	}
 
-		// save does not touch any changing data
-		// must not be locked, will deadlock with cleaner functions
-		if conn.saveWhenFinished {
-			conn.saveWhenFinished = false
-			conn.Save()
-		}
+	// Log verdict.
+	log.Tracer(pkt.Ctx()).Infof("filter: connection %s %s: %s", conn, conn.Verdict.Verb(), conn.Reason.Msg)
+	// Submit trace logs.
+	log.Tracer(pkt.Ctx()).Submit()
 
-		conn.Unlock()
+	// Save() itself does not touch any changing data.
+	// Must not be locked - would deadlock with cleaner functions.
+	if conn.saveWhenFinished {
+		conn.saveWhenFinished = false
+		conn.Save()
 	}
 }
 

From 7d2ae6823722852893772efae536ab009a2e03a6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 31 Mar 2022 10:02:14 +0200
Subject: [PATCH 07/86] Improve prevented bypass attempt notification

---
 compat/notify.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compat/notify.go b/compat/notify.go
index 9a263e7a..8fa64577 100644
--- a/compat/notify.go
+++ b/compat/notify.go
@@ -44,7 +44,7 @@ var (
 
 	secureDNSBypassIssue = &appIssue{
 		id:    "compat:secure-dns-bypass-%s",
-		title: "Prevented %s Bypass Attempt",
+		title: "Prevented Bypass Attempt by %s",
 		message: `[APPNAME] attempted to bypass Portmaster's firewall functions by directly using a Secure DNS resolver. If [APPNAME] experiences problems, disable Secure DNS within [APPNAME] to restore functionality. Rest assured that Portmaster already handles Secure DNS for your whole device, including [APPNAME].  
 If bypass prevention for [APPNAME] is disabled in Portmaster, it can no longer protect or filter connections coming from [APPNAME].`,
 		// TODO: Add this when the new docs page is finished:

From be95bc98690be3c0c02ad00c61490d4e1fef63f0 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 7 Apr 2022 16:08:30 +0200
Subject: [PATCH 08/86] Raise dns/nameservers config to expertise level user

---
 resolver/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resolver/config.go b/resolver/config.go
index 8e9e8109..b274ee23 100644
--- a/resolver/config.go
+++ b/resolver/config.go
@@ -109,7 +109,7 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 `, `"`, "`"),
 		Sensitive:       true,
 		OptType:         config.OptTypeStringArray,
-		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ExpertiseLevel:  config.ExpertiseLevelUser,
 		ReleaseLevel:    config.ReleaseLevelStable,
 		DefaultValue:    defaultNameServers,
 		ValidationRegex: fmt.Sprintf("^(%s|%s|%s)://.*", ServerTypeDoT, ServerTypeDNS, ServerTypeTCP),

From c6ffa51fd7db27489a99a0bfd8330cd7056367c6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:36:47 +0200
Subject: [PATCH 09/86] Improve wording on block bypassing notification and
 setting

---
 compat/notify.go  | 7 +++----
 profile/config.go | 4 +++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/compat/notify.go b/compat/notify.go
index 8fa64577..04beeff7 100644
--- a/compat/notify.go
+++ b/compat/notify.go
@@ -43,10 +43,9 @@ var (
 	}
 
 	secureDNSBypassIssue = &appIssue{
-		id:    "compat:secure-dns-bypass-%s",
-		title: "Prevented Bypass Attempt by %s",
-		message: `[APPNAME] attempted to bypass Portmaster's firewall functions by directly using a Secure DNS resolver. If [APPNAME] experiences problems, disable Secure DNS within [APPNAME] to restore functionality. Rest assured that Portmaster already handles Secure DNS for your whole device, including [APPNAME].  
-If bypass prevention for [APPNAME] is disabled in Portmaster, it can no longer protect or filter connections coming from [APPNAME].`,
+		id:      "compat:secure-dns-bypass-%s",
+		title:   "Blocked Bypass Attempt by %s",
+		message: `[APPNAME] is using its own Secure DNS resolver, which would bypass Portmaster's firewall protections. If [APPNAME] experiences problems, disable Secure DNS within [APPNAME] to restore functionality. Rest assured that Portmaster handles Secure DNS for your whole device, including [APPNAME].`,
 		// TODO: Add this when the new docs page is finished:
 		// , or [find out about other options](link to new docs page)
 		level: notifications.Warning,
diff --git a/profile/config.go b/profile/config.go
index b89f3777..987594d4 100644
--- a/profile/config.go
+++ b/profile/config.go
@@ -575,7 +575,9 @@ The lists are automatically updated every hour using incremental updates.
 	err = config.Register(&config.Option{
 		Name: "Block Bypassing",
 		Key:  CfgOptionPreventBypassingKey,
-		Description: `Prevent apps from bypassing the privacy filter.  
+		Description: `Prevent apps from bypassing Portmaster's privacy protections.  
+If Block Bypassing is disabled, Portmaster can no longer protect you or filter connections from the affected applications.
+
 Current Features:  
 - Disable Firefox' internal DNS-over-HTTPs resolver
 - Block direct access to public DNS resolvers

From cb9d8cee06de3ab4fa6ef9538c9c11216ac56c79 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:38:50 +0200
Subject: [PATCH 10/86] Only warn when registry cleanup fails

---
 updates/main.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/updates/main.go b/updates/main.go
index 0352f54b..550e23ac 100644
--- a/updates/main.go
+++ b/updates/main.go
@@ -269,7 +269,10 @@ func checkForUpdates(ctx context.Context) (err error) {
 
 func stop() error {
 	if registry != nil {
-		return registry.Cleanup()
+		err := registry.Cleanup()
+		if err != nil {
+			log.Warningf("updates: failed to clean up registry: %s", err)
+		}
 	}
 
 	return stopVersionExport()

From 295efbb3a205086032d608969547e5b0d51cc2c0 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:39:42 +0200
Subject: [PATCH 11/86] Ignore comments on rule parsing

---
 profile/endpoints/endpoint.go  | 8 ++++++++
 profile/endpoints/endpoints.go | 4 +++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/profile/endpoints/endpoint.go b/profile/endpoints/endpoint.go
index 39f9700a..63c1e8b3 100644
--- a/profile/endpoints/endpoint.go
+++ b/profile/endpoints/endpoint.go
@@ -208,6 +208,14 @@ func parseEndpoint(value string) (endpoint Endpoint, err error) { //nolint:gocog
 		return nil, fmt.Errorf(`invalid endpoint definition: "%s"`, value)
 	}
 
+	// Remove comment.
+	for i, field := range fields {
+		if strings.HasPrefix(field, "#") {
+			fields = fields[:i]
+			break
+		}
+	}
+
 	// any
 	if endpoint, err = parseTypeAny(fields); endpoint != nil || err != nil {
 		return
diff --git a/profile/endpoints/endpoints.go b/profile/endpoints/endpoints.go
index 7b16fab5..05273f95 100644
--- a/profile/endpoints/endpoints.go
+++ b/profile/endpoints/endpoints.go
@@ -62,11 +62,13 @@ entriesLoop:
 // ListEntryValidationRegex is a regex to bullshit check endpoint list entries.
 var ListEntryValidationRegex = strings.Join([]string{
 	`^(\+|\-) `,                   // Rule verdict.
+	`(! +)?`,                      // Invert matching.
 	`[A-z0-9\.:\-*/]+`,            // Entity matching.
 	`( `,                          // Start of optional matching.
 	`[A-z0-9*]+`,                  // Protocol matching.
 	`(/[A-z0-9]+(\-[A-z0-9]+)?)?`, // Port and port range matching.
-	`)?$`,                         // End of optional matching.
+	`)?`,                          // End of optional matching.
+	`( +#.*)?`,                    // Optional comment.
 }, "")
 
 // ValidateEndpointListConfigOption validates the given value.

From d89609e7e757ff6a6054bbeb55a7193051c0ecc4 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:40:29 +0200
Subject: [PATCH 12/86] Add intel-only flag for initializing intel data only

---
 cmds/portmaster-start/update.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cmds/portmaster-start/update.go b/cmds/portmaster-start/update.go
index 45b7df8e..3f70bfc3 100644
--- a/cmds/portmaster-start/update.go
+++ b/cmds/portmaster-start/update.go
@@ -11,7 +11,10 @@ import (
 	"github.com/safing/portmaster/updates/helper"
 )
 
-var reset bool
+var (
+	reset     bool
+	intelOnly bool
+)
 
 func init() {
 	rootCmd.AddCommand(updateCmd)
@@ -19,6 +22,7 @@ func init() {
 
 	flags := updateCmd.Flags()
 	flags.BoolVar(&reset, "reset", false, "Delete all resources and re-download the basic set")
+	flags.BoolVar(&intelOnly, "intel-only", false, "Only make downloading intel updates mandatory")
 }
 
 var (
@@ -49,6 +53,11 @@ func indexRequired(cmd *cobra.Command) bool {
 }
 
 func downloadUpdates() error {
+	// Check if only intel data is mandatory.
+	if intelOnly {
+		helper.IntelOnly()
+	}
+
 	// Set required updates.
 	registry.MandatoryUpdates = helper.MandatoryUpdates()
 	registry.AutoUnpack = helper.AutoUnpackUpdates()

From 650c272301e0f97eb113c003c8fc2f2c7b5aed69 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:56:33 +0200
Subject: [PATCH 13/86] Only fix chrome sandbox permission when needed

---
 cmds/portmaster-start/update.go | 8 +++++---
 updates/helper/electron.go      | 4 ++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cmds/portmaster-start/update.go b/cmds/portmaster-start/update.go
index 3f70bfc3..e57b9b69 100644
--- a/cmds/portmaster-start/update.go
+++ b/cmds/portmaster-start/update.go
@@ -106,9 +106,11 @@ func downloadUpdates() error {
 		return fmt.Errorf("failed to unpack resources: %w", err)
 	}
 
-	// Fix chrome-sandbox permissions
-	if err := helper.EnsureChromeSandboxPermissions(registry); err != nil {
-		return fmt.Errorf("failed to fix electron permissions: %w", err)
+	if !intelOnly {
+		// Fix chrome-sandbox permissions
+		if err := helper.EnsureChromeSandboxPermissions(registry); err != nil {
+			return fmt.Errorf("failed to fix electron permissions: %w", err)
+		}
 	}
 
 	return nil
diff --git a/updates/helper/electron.go b/updates/helper/electron.go
index ca9ba1b7..833c1c91 100644
--- a/updates/helper/electron.go
+++ b/updates/helper/electron.go
@@ -1,6 +1,7 @@
 package helper
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -34,6 +35,9 @@ func EnsureChromeSandboxPermissions(reg *updater.ResourceRegistry) error {
 	var err error
 	pmElectronUpdate, err = reg.GetFile(identifier)
 	if err != nil {
+		if errors.Is(err, updater.ErrNotAvailableLocally) {
+			return nil
+		}
 		return fmt.Errorf("failed to get file: %w", err)
 	}
 

From 014eb95eacd154720cec70c2e176c128cf9727ee Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 13 Apr 2022 11:04:08 +0200
Subject: [PATCH 14/86] Reset device locations when network changes

---
 netenv/location.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/netenv/location.go b/netenv/location.go
index 75da8c87..6245ab45 100644
--- a/netenv/location.go
+++ b/netenv/location.go
@@ -216,6 +216,10 @@ func SetInternetLocation(ip net.IP, source DeviceLocationSource) (dl *DeviceLoca
 		log.Warningf("netenv: failed to get geolocation data of %s (from %s): %s", ip, source, err)
 		return nil, false
 	}
+	// Only use location if there is data for it.
+	if geoLoc.Country.ISOCode == "" {
+		return nil, false
+	}
 	loc.Location = geoLoc
 
 	addLocation(loc)
@@ -271,6 +275,13 @@ func GetInternetLocation() (deviceLocations *DeviceLocations, ok bool) {
 	}
 	locationNetworkChangedFlag.Refresh()
 
+	// Reset locations.
+	func() {
+		locationsLock.Lock()
+		defer locationsLock.Unlock()
+		locations = &DeviceLocations{}
+	}()
+
 	// Get all assigned addresses.
 	v4s, v6s, err := GetAssignedAddresses()
 	if err != nil {

From fdafc1ea5e1767e043bf637d09a10af0fd871e4e Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 13 Apr 2022 11:05:39 +0200
Subject: [PATCH 15/86] Improve IP scope classification

---
 network/netutils/ip.go | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/network/netutils/ip.go b/network/netutils/ip.go
index 0cb8f59f..f6a34003 100644
--- a/network/netutils/ip.go
+++ b/network/netutils/ip.go
@@ -29,29 +29,43 @@ func GetIPScope(ip net.IP) IPScope { //nolint:gocognit
 		// IPv4
 		switch {
 		case ip4[0] == 127:
-			// 127.0.0.0/8
+			// 127.0.0.0/8 (RFC1918)
 			return HostLocal
 		case ip4[0] == 169 && ip4[1] == 254:
-			// 169.254.0.0/16
+			// 169.254.0.0/16 (RFC3927)
 			return LinkLocal
 		case ip4[0] == 10:
-			// 10.0.0.0/8
+			// 10.0.0.0/8 (RFC1918)
 			return SiteLocal
-		case ip4[0] == 172 && ip4[1]&0xf0 == 16:
-			// 172.16.0.0/12
+		case ip4[0] == 100 && ip4[1]&0b11000000 == 64:
+			// 100.64.0.0/10 (RFC6598)
 			return SiteLocal
+		case ip4[0] == 172 && ip4[1]&0b11110000 == 16:
+			// 172.16.0.0/12 (RFC1918)
+			return SiteLocal
+		case ip4[0] == 192 && ip4[1] == 0 && ip4[2] == 2:
+			// 192.0.2.0/24 (TEST-NET-1, RFC5737)
+			return Invalid
 		case ip4[0] == 192 && ip4[1] == 168:
-			// 192.168.0.0/16
+			// 192.168.0.0/16 (RFC1918)
 			return SiteLocal
+		case ip4[0] == 198 && ip4[1] == 51 && ip4[2] == 100:
+			// 198.51.100.0/24 (TEST-NET-2, RFC5737)
+			return Invalid
+		case ip4[0] == 203 && ip4[1] == 0 && ip4[2] == 113:
+			// 203.0.113.0/24 (TEST-NET-3, RFC5737)
+			return Invalid
 		case ip4[0] == 224:
-			// 224.0.0.0/8
+			// 224.0.0.0/8 (RFC5771)
 			return LocalMulticast
+		case ip4[0] == 233 && ip4[1] == 252 && ip4[2] == 0:
+			// 233.252.0.0/24 (MCAST-TEST-NET; RFC5771, RFC6676)
+			return Invalid
 		case ip4[0] >= 225 && ip4[0] <= 238:
-			// 225.0.0.0/8 - 238.0.0.0/8
+			// 225.0.0.0/8 - 238.0.0.0/8 (RFC5771)
 			return GlobalMulticast
 		case ip4[0] == 239:
-			// 239.0.0.0/8
-			// RFC2365 - https://tools.ietf.org/html/rfc2365
+			// 239.0.0.0/8 (RFC2365)
 			return LocalMulticast
 		case ip4[0] == 255 && ip4[1] == 255 && ip4[2] == 255 && ip4[3] == 255:
 			// 255.255.255.255/32

From fc3c1ccdbb3ec922d235461076e26ee398d497e4 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 13 Apr 2022 11:55:49 +0200
Subject: [PATCH 16/86] Update deps

---
 go.mod | 17 +++++++----------
 go.sum | 32 ++++++++++++++++++++------------
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/go.mod b/go.mod
index d41c5211..cff7a842 100644
--- a/go.mod
+++ b/go.mod
@@ -11,20 +11,17 @@ require (
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.4.0
-	github.com/mdlayher/socket v0.2.3 // indirect
-	github.com/miekg/dns v1.1.47
-	github.com/oschwald/maxminddb-golang v1.8.0
-	github.com/safing/portbase v0.14.1
-	github.com/safing/spn v0.4.5
+	github.com/miekg/dns v1.1.48
+	github.com/oschwald/maxminddb-golang v1.9.0
+	github.com/safing/portbase v0.14.2
+	github.com/safing/spn v0.4.6
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220412020605-290c469a71a5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
-	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
 )
diff --git a/go.sum b/go.sum
index 57dbf99b..e249db8a 100644
--- a/go.sum
+++ b/go.sum
@@ -686,8 +686,9 @@ github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0
 github.com/miekg/dns v1.1.44/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
-github.com/miekg/dns v1.1.47 h1:J9bWiXbqMbnZPcY8Qi2E3EWIBsIm6MZzzJB9VRg5gL8=
 github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
+github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -743,8 +744,9 @@ github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
 github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
+github.com/oschwald/maxminddb-golang v1.9.0 h1:tIk4nv6VT9OiPyrnDAfJS1s1xKDQMZOsGojab6EjC1Y=
+github.com/oschwald/maxminddb-golang v1.9.0/go.mod h1:TK+s/Z2oZq0rSl4PSeAEoP0bgm82Cp5HyvYbt8K3zLY=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -812,8 +814,9 @@ github.com/safing/portbase v0.13.4/go.mod h1:5vj5IK5WJoSGareDe6yCMZfnF7txVRx7jZy
 github.com/safing/portbase v0.13.5/go.mod h1:5vj5IK5WJoSGareDe6yCMZfnF7txVRx7jZyTZInISP0=
 github.com/safing/portbase v0.13.6/go.mod h1:G0maDSQxYDuluNhMzA1zVd/nfXawfECv5H7+fnTfVhM=
 github.com/safing/portbase v0.14.0/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
-github.com/safing/portbase v0.14.1 h1:k/SgywE5QutDbf5hHbEArrQnjBByerGm0nhh5n8VnDw=
 github.com/safing/portbase v0.14.1/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.2 h1:xL1TebfvWsSbf5AsZxdY0+/+mArgkqmSnmnqsRZpZYw=
+github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=
@@ -828,6 +831,7 @@ github.com/safing/portmaster v0.7.21/go.mod h1:Jy0G6x6m5dE36Mv9grXHI77cxysQ0fIQV
 github.com/safing/portmaster v0.8.0/go.mod h1:lY2/WvOlH8kl1AwkixdWCjlo+PZQv+oEOQhIaSS/+wA=
 github.com/safing/portmaster v0.8.5-interdep/go.mod h1:A+zAVEKjr057ktgiMSJRdUmOF+FPW8XY/5LqGnbsKbU=
 github.com/safing/portmaster v0.8.5/go.mod h1:MqOlFwHcIx/109Ugutz/CG23znuuXCRVHcIcfX0VC/c=
+github.com/safing/portmaster v0.8.7/go.mod h1:RUgCWt5v22jDUOtJfOwApi//Kt8RTZQhlREcBc+L4z8=
 github.com/safing/spn v0.3.4/go.mod h1:TfzNsZCbnlWv0UFDILFOUSudVKJZlnBVoR1fDXrjOK0=
 github.com/safing/spn v0.3.5/go.mod h1:jHkFF2Yu1fnjFu4KXjVA+iagMr/z4eB4p3jiwikvKj8=
 github.com/safing/spn v0.3.6/go.mod h1:RSeFb/h5Wt3yDVezXj3lhXJ/Iwd7FbtsGf5E+p5J2YQ=
@@ -841,12 +845,10 @@ github.com/safing/spn v0.3.17/go.mod h1:Fq/70Hl0OUxtYuY5NATv5q468hvfDDEFwN3mivEe
 github.com/safing/spn v0.3.19/go.mod h1:phCnWjWOgdVMXaMsmDr6izR/ROVElSZGdIm7j7PIit4=
 github.com/safing/spn v0.4.0/go.mod h1:0jBetnYCfxqO5PJskhPOxJ/v6VRfE+bQU98XW240BNw=
 github.com/safing/spn v0.4.2/go.mod h1:yZPezHDEYyhei8n13tTxjQCGq6LRr5svz9WFAAeDPec=
-github.com/safing/spn v0.4.3 h1:iEFmpzyrThJ8QF9Qpbxk/m4w2+ZvbVPyuqJ4EwnpfDg=
 github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW2M=
-github.com/safing/spn v0.4.4 h1:DGCkHaCgkQ0ivsD3J3BkUJ2auMpCEhzZdlD3JDmK/U4=
-github.com/safing/spn v0.4.4/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
-github.com/safing/spn v0.4.5 h1:I3nv0YOD7Rh+hTpWHdsIlilK10Dz/ZZF//OfJm+Un3I=
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
+github.com/safing/spn v0.4.6 h1:jOxFdVpSSXCCCiNktBzPNfom6Tf1qtsjr2USFiTGHIM=
+github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -913,8 +915,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tannerryan/ring v1.1.2 h1:iXayOjqHQOLzuy9GwSKuG3nhWfzQkldMlQivcgIr7gQ=
 github.com/tannerryan/ring v1.1.2/go.mod h1:DkELJEjbZhJBtFKR9Xziwj3HKZnb/knRgljNqp65vH4=
@@ -1072,8 +1075,9 @@ golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220213190939-1e6e3497d506/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s=
 golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1190,8 +1194,9 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1344,8 +1349,10 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1446,8 +1453,9 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

From ce99f10038cf09df92c8da33cfe6b1eb295e24e0 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 13 Apr 2022 11:55:59 +0200
Subject: [PATCH 17/86] Bump versions

---
 cmds/portmaster-core/main.go  | 2 +-
 cmds/portmaster-start/main.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index 7829ba2a..27dd5175 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.7", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.8", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")
diff --git a/cmds/portmaster-start/main.go b/cmds/portmaster-start/main.go
index a953189b..6afc831d 100644
--- a/cmds/portmaster-start/main.go
+++ b/cmds/portmaster-start/main.go
@@ -77,7 +77,7 @@ func main() {
 	cobra.OnInitialize(initCobra)
 
 	// set meta info
-	info.Set("Portmaster Start", "0.8.6", "AGPLv3", false)
+	info.Set("Portmaster Start", "0.8.8", "AGPLv3", false)
 
 	// catch interrupt for clean shutdown
 	signalCh := make(chan os.Signal, 2)

From 29bfa9fd916af5d1c3428a80a4d8caeb6c9f2506 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 15 Apr 2022 13:05:24 +0200
Subject: [PATCH 18/86] Add config option to disable dns query interception

---
 firewall/config.go       | 22 ++++++++++++++++++++++
 firewall/filter.go       |  2 --
 firewall/interception.go |  9 ++++-----
 go.mod                   |  2 +-
 4 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/firewall/config.go b/firewall/config.go
index 2e283a49..702b6c95 100644
--- a/firewall/config.go
+++ b/firewall/config.go
@@ -23,6 +23,10 @@ var (
 	cfgOptionPermanentVerdictsOrder = 96
 	permanentVerdicts               config.BoolOption
 
+	CfgOptionDNSQueryInterceptionKey   = "filter/dnsQueryInterception"
+	cfgOptionDNSQueryInterceptionOrder = 97
+	dnsQueryInterception               config.BoolOption
+
 	devMode          config.BoolOption
 	apiListenAddress config.StringOption
 )
@@ -46,6 +50,24 @@ func registerConfig() error {
 	}
 	permanentVerdicts = config.Concurrent.GetAsBool(CfgOptionPermanentVerdictsKey, true)
 
+	err = config.Register(&config.Option{
+		Name:           "Seamless DNS Integration",
+		Key:            CfgOptionDNSQueryInterceptionKey,
+		Description:    "Intercept and redirect astray DNS queries to the Portmaster's internal DNS server. This enables seamless DNS integration without having to configure the system or other software. However, this may lead to compatibility issues with other software that attempts the same.",
+		OptType:        config.OptTypeBool,
+		ExpertiseLevel: config.ExpertiseLevelDeveloper,
+		ReleaseLevel:   config.ReleaseLevelExperimental,
+		DefaultValue:   true,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionDNSQueryInterceptionOrder,
+			config.CategoryAnnotation:     "Advanced",
+		},
+	})
+	if err != nil {
+		return err
+	}
+	dnsQueryInterception = config.Concurrent.GetAsBool(CfgOptionDNSQueryInterceptionKey, true)
+
 	err = config.Register(&config.Option{
 		Name:           "Prompt Desktop Notifications",
 		Key:            CfgOptionAskWithSystemNotificationsKey,
diff --git a/firewall/filter.go b/firewall/filter.go
index cb7dc3aa..58a24737 100644
--- a/firewall/filter.go
+++ b/firewall/filter.go
@@ -4,8 +4,6 @@ import (
 	"github.com/safing/portbase/config"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
-
-	// Dependency.
 	_ "github.com/safing/portmaster/core"
 	"github.com/safing/spn/captain"
 )
diff --git a/firewall/interception.go b/firewall/interception.go
index 25e0354b..ed582ba1 100644
--- a/firewall/interception.go
+++ b/firewall/interception.go
@@ -16,8 +16,6 @@ import (
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portmaster/compat"
-
-	// Dependency.
 	_ "github.com/safing/portmaster/core/base"
 	"github.com/safing/portmaster/firewall/inspection"
 	"github.com/safing/portmaster/firewall/interception"
@@ -332,8 +330,9 @@ func initialHandler(conn *network.Connection, pkt packet.Packet) {
 		conn.Accept("connection by Portmaster", noReasonOptionKey)
 		conn.Internal = true
 
-		// Redirect outbound DNS packests,
-	case pkt.IsOutbound() &&
+		// Redirect outbound DNS packets if enabled,
+	case dnsQueryInterception() &&
+		pkt.IsOutbound() &&
 		pkt.Info().DstPort == 53 &&
 		// that don't match the address of our nameserver,
 		nameserverIPMatcherReady.IsSet() &&
@@ -341,7 +340,7 @@ func initialHandler(conn *network.Connection, pkt packet.Packet) {
 		// and are not broadcast queries by us.
 		// Context:
 		// - Unicast queries by the resolver are pre-authenticated.
-		// - Unicast qeries by the compat self-check should be redirected.
+		// - Unicast queries by the compat self-check should be redirected.
 		!(conn.Process().Pid == ownPID &&
 			conn.Entity.IPScope == netutils.LocalMulticast):
 
diff --git a/go.mod b/go.mod
index cff7a842..b128b856 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/safing/portmaster
 
-go 1.15
+go 1.18
 
 require (
 	github.com/agext/levenshtein v1.2.3

From f5afe8b5dfa1daa1e9e95bc51190f598cf4b4937 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 15 Apr 2022 13:06:13 +0200
Subject: [PATCH 19/86] Block DNS requests if bypass prevention is active

---
 firewall/bypassing.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/firewall/bypassing.go b/firewall/bypassing.go
index 4f6b0f1d..cf8502cb 100644
--- a/firewall/bypassing.go
+++ b/firewall/bypassing.go
@@ -43,8 +43,12 @@ func PreventBypassing(ctx context.Context, conn *network.Connection) (endpoints.
 		return endpoints.NoMatch, "", nil
 	}
 
-	// Block bypass attempts using an encrypted DNS server.
+	// Block bypass attempts using an (encrypted) DNS server.
 	switch {
+	case conn.Entity.Port == 53:
+		return endpoints.Denied,
+			"blocked DNS query, manual dns setup required",
+			nsutil.BlockIP()
 	case conn.Entity.Port == 853:
 		// Block connections to port 853 - DNS over TLS.
 		fallthrough

From 2c3def3bc43414e82e78d1500c97f0c99b575c89 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 15 Apr 2022 13:06:59 +0200
Subject: [PATCH 20/86] Show notification about manual DNS setup instead of
 compatibility notice

---
 compat/module.go         | 14 +++++--
 compat/notify.go         | 90 +++++++++++++++++++++++++++++++++-------
 compat/selfcheck.go      | 16 +++----
 nameserver/module.go     |  4 ++
 nameserver/nameserver.go |  2 +-
 5 files changed, 98 insertions(+), 28 deletions(-)

diff --git a/compat/module.go b/compat/module.go
index 1517f2cc..d1a27628 100644
--- a/compat/module.go
+++ b/compat/module.go
@@ -16,7 +16,7 @@ var (
 	module *modules.Module
 
 	selfcheckTask           *modules.Task
-	selfcheckTaskRetryAfter = 10 * time.Second
+	selfcheckTaskRetryAfter = 5 * time.Second
 
 	// selfCheckIsFailing holds whether or not the self-check is currently
 	// failing. This helps other failure systems to not make noise when there is
@@ -47,6 +47,8 @@ func prep() error {
 }
 
 func start() error {
+	startNotify()
+
 	selfcheckTask = module.NewTask("compatibility self-check", selfcheckTaskFunc).
 		Repeat(5 * time.Minute).
 		MaxDelay(selfcheckTaskRetryAfter).
@@ -74,12 +76,18 @@ func stop() error {
 }
 
 func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
+	// Create tracing logger.
+	ctx, tracer := log.AddTracer(ctx)
+	defer tracer.Submit()
+	tracer.Tracef("compat: running self-check")
+
 	// Run selfcheck and return if successful.
 	issue, err := selfcheck(ctx)
 	if err == nil {
 		selfCheckIsFailing.UnSet()
 		selfcheckFails = 0
 		resetSystemIssue()
+		tracer.Debugf("compat: self-check successful")
 		return nil
 	}
 
@@ -88,7 +96,7 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 		selfCheckIsFailing.Set()
 		selfcheckFails++
 
-		log.Errorf("compat: %s", err)
+		tracer.Errorf("compat: %s", err)
 		if selfcheckFails >= selfcheckFailThreshold {
 			issue.notify(err)
 		}
@@ -100,7 +108,7 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 		selfcheckFails = 0
 
 		// Only log internal errors, but don't notify.
-		log.Warningf("compat: %s", err)
+		tracer.Warningf("compat: %s", err)
 	}
 
 	return nil
diff --git a/compat/notify.go b/compat/notify.go
index 04beeff7..b483ead5 100644
--- a/compat/notify.go
+++ b/compat/notify.go
@@ -3,10 +3,12 @@ package compat
 import (
 	"context"
 	"fmt"
+	"net"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/safing/portbase/config"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/notifications"
@@ -15,10 +17,11 @@ import (
 )
 
 type baseIssue struct {
-	id      string             //nolint:structcheck // Inherited.
-	title   string             //nolint:structcheck // Inherited.
-	message string             //nolint:structcheck // Inherited.
-	level   notifications.Type //nolint:structcheck // Inherited.
+	id      string                  //nolint:structcheck // Inherited.
+	title   string                  //nolint:structcheck // Inherited.
+	message string                  //nolint:structcheck // Inherited.
+	level   notifications.Type      //nolint:structcheck // Inherited.
+	actions []*notifications.Action //nolint:structcheck // Inherited.
 }
 
 type systemIssue baseIssue
@@ -26,6 +29,10 @@ type systemIssue baseIssue
 type appIssue baseIssue
 
 var (
+	// Copy of firewall.CfgOptionDNSQueryInterceptionKey.
+	cfgOptionDNSQueryInterceptionKey = "filter/dnsQueryInterception"
+	dnsQueryInterception             config.BoolOption
+
 	systemIssueNotification     *notifications.Notification
 	systemIssueNotificationLock sync.Mutex
 
@@ -41,6 +48,22 @@ var (
 		message: "Portmaster detected that something is interfering with its operation. This could be a VPN, an Anti-Virus or another network protection software. Please check if you are running an incompatible [VPN client](https://docs.safing.io/portmaster/install/status/vpn-compatibility) or [software](https://docs.safing.io/portmaster/install/status/software-compatibility). Otherwise, please report the issue via [GitHub](https://github.com/safing/portmaster/issues) or send a mail to [support@safing.io](mailto:support@safing.io) so we can help you out.",
 		level:   notifications.Error,
 	}
+	// manualDNSSetupRequired is additionally initialized in startNotify().
+	manualDNSSetupRequired = &systemIssue{
+		id:    "compat:manual-dns-setup-required",
+		title: "Manual DNS Setup Required",
+		level: notifications.Error,
+		actions: []*notifications.Action{
+			{
+				Text: "Revert",
+				Type: notifications.ActionTypeOpenSetting,
+				Payload: &notifications.ActionTypeOpenSettingPayload{
+					Key: cfgOptionDNSQueryInterceptionKey,
+				},
+			},
+		},
+	}
+	manualDNSSetupRequiredMessage = "You have disabled Seamless DNS Integration. As a result, Portmaster can no longer protect you or filter connections reliably. To fix this, you have to manually configure %s as the DNS Server in your system and in any conflicting application. This message will disappear 10 seconds after correct configuration."
 
 	secureDNSBypassIssue = &appIssue{
 		id:      "compat:secure-dns-bypass-%s",
@@ -58,6 +81,37 @@ var (
 	}
 )
 
+func startNotify() {
+	dnsQueryInterception = config.Concurrent.GetAsBool(cfgOptionDNSQueryInterceptionKey, true)
+
+	systemIssueNotificationLock.Lock()
+	defer systemIssueNotificationLock.Unlock()
+
+	manualDNSSetupRequired.message = fmt.Sprintf(
+		manualDNSSetupRequiredMessage,
+		`"127.0.0.1"`,
+	)
+}
+
+// SetNameserverListenIP sets the IP address the nameserver is listening on.
+// The IP address is used in compatibility notifications.
+func SetNameserverListenIP(ip net.IP) {
+	systemIssueNotificationLock.Lock()
+	defer systemIssueNotificationLock.Unlock()
+
+	manualDNSSetupRequired.message = fmt.Sprintf(
+		manualDNSSetupRequiredMessage,
+		`"`+ip.String()+`"`,
+	)
+}
+
+func systemCompatOrManualDNSIssue() *systemIssue {
+	if dnsQueryInterception() {
+		return systemCompatibilityIssue
+	}
+	return manualDNSSetupRequired
+}
+
 func (issue *systemIssue) notify(err error) {
 	systemIssueNotificationLock.Lock()
 	defer systemIssueNotificationLock.Unlock()
@@ -74,11 +128,12 @@ func (issue *systemIssue) notify(err error) {
 
 	// Create new notification.
 	n := &notifications.Notification{
-		EventID:      issue.id,
-		Type:         issue.level,
-		Title:        issue.title,
-		Message:      issue.message,
-		ShowOnSystem: true,
+		EventID:          issue.id,
+		Type:             issue.level,
+		Title:            issue.title,
+		Message:          issue.message,
+		ShowOnSystem:     true,
+		AvailableActions: issue.actions,
 	}
 	notifications.Notify(n)
 
@@ -141,17 +196,20 @@ func (issue *appIssue) notify(proc *process.Process) {
 
 	// Create a new notification.
 	n = &notifications.Notification{
-		EventID:      eventID,
-		Type:         issue.level,
-		Title:        fmt.Sprintf(issue.title, p.Name),
-		Message:      message,
-		ShowOnSystem: true,
-		AvailableActions: []*notifications.Action{
+		EventID:          eventID,
+		Type:             issue.level,
+		Title:            fmt.Sprintf(issue.title, p.Name),
+		Message:          message,
+		ShowOnSystem:     true,
+		AvailableActions: issue.actions,
+	}
+	if len(n.AvailableActions) == 0 {
+		n.AvailableActions = []*notifications.Action{
 			{
 				ID:   "ack",
 				Text: "OK",
 			},
-		},
+		}
 	}
 	notifications.Notify(n)
 
diff --git a/compat/selfcheck.go b/compat/selfcheck.go
index 1931b70e..c1508d12 100644
--- a/compat/selfcheck.go
+++ b/compat/selfcheck.go
@@ -28,12 +28,12 @@ var (
 	systemIntegrationCheckDialNet      = fmt.Sprintf("ip4:%d", uint8(SystemIntegrationCheckProtocol))
 	systemIntegrationCheckDialIP       = SystemIntegrationCheckDstIP.String()
 	systemIntegrationCheckPackets      = make(chan packet.Packet, 1)
-	systemIntegrationCheckWaitDuration = 30 * time.Second
+	systemIntegrationCheckWaitDuration = 20 * time.Second
 
 	// DNSCheckInternalDomainScope is the domain scope to use for dns checks.
 	DNSCheckInternalDomainScope = ".self-check." + resolver.InternalSpecialUseDomain
 	dnsCheckReceivedDomain      = make(chan string, 1)
-	dnsCheckWaitDuration        = 30 * time.Second
+	dnsCheckWaitDuration        = 20 * time.Second
 	dnsCheckAnswerLock          sync.Mutex
 	dnsCheckAnswer              net.IP
 )
@@ -61,7 +61,7 @@ func selfcheck(ctx context.Context) (issue *systemIssue, err error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create system integration conn: %w", err)
 	}
-	_, err = conn.Write([]byte("SELF-CHECK"))
+	_, err = conn.Write([]byte("PORTMASTER SELF CHECK"))
 	if err != nil {
 		return nil, fmt.Errorf("failed to send system integration packet: %w", err)
 	}
@@ -70,7 +70,7 @@ func selfcheck(ctx context.Context) (issue *systemIssue, err error) {
 	select {
 	case <-systemIntegrationCheckPackets:
 		// Check passed!
-		log.Tracef("compat: self-check #1: system integration check passed")
+		log.Tracer(ctx).Tracef("compat: self-check #1: system integration check passed")
 	case <-time.After(systemIntegrationCheckWaitDuration):
 		return systemIntegrationIssue, fmt.Errorf("self-check #1: system integration check failed: did not receive test packet after %s", systemIntegrationCheckWaitDuration)
 	case <-ctx.Done():
@@ -139,12 +139,12 @@ func selfcheck(ctx context.Context) (issue *systemIssue, err error) {
 	select {
 	case receivedTestDomain := <-dnsCheckReceivedDomain:
 		if receivedTestDomain != randomSubdomain {
-			return systemCompatibilityIssue, fmt.Errorf("self-check #2: dns integration check failed: received unmatching subdomain %q", receivedTestDomain)
+			return systemCompatOrManualDNSIssue(), fmt.Errorf("self-check #2: dns integration check failed: received unmatching subdomain %q", receivedTestDomain)
 		}
 	case <-time.After(dnsCheckWaitDuration):
-		return systemCompatibilityIssue, fmt.Errorf("self-check #2: dns integration check failed: did not receive test query after %s", dnsCheckWaitDuration)
+		return systemCompatOrManualDNSIssue(), fmt.Errorf("self-check #2: dns integration check failed: did not receive test query after %s", dnsCheckWaitDuration)
 	}
-	log.Tracef("compat: self-check #2: dns integration query check passed")
+	log.Tracer(ctx).Tracef("compat: self-check #2: dns integration query check passed")
 
 	// Step 3: Have the nameserver respond with random data in the answer section.
 
@@ -164,7 +164,7 @@ func selfcheck(ctx context.Context) (issue *systemIssue, err error) {
 	if !dnsCheckReturnedIP.Equal(randomAnswer) {
 		return systemCompatibilityIssue, fmt.Errorf("self-check #3: dns integration check failed: received unmatching response %q", dnsCheckReturnedIP)
 	}
-	log.Tracef("compat: self-check #3: dns integration response check passed")
+	log.Tracer(ctx).Tracef("compat: self-check #3: dns integration response check passed")
 
 	return nil, nil
 }
diff --git a/nameserver/module.go b/nameserver/module.go
index 7bc177e4..1c1dc871 100644
--- a/nameserver/module.go
+++ b/nameserver/module.go
@@ -13,6 +13,7 @@ import (
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
+	"github.com/safing/portmaster/compat"
 	"github.com/safing/portmaster/firewall"
 	"github.com/safing/portmaster/netenv"
 )
@@ -53,6 +54,9 @@ func start() error {
 		return fmt.Errorf("failed to parse nameserver listen address: %w", err)
 	}
 
+	// Tell the compat module where we are listening.
+	compat.SetNameserverListenIP(ip1)
+
 	// Get own hostname.
 	hostname, err = os.Hostname()
 	if err != nil {
diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go
index 4c5bf114..a92ca8ad 100644
--- a/nameserver/nameserver.go
+++ b/nameserver/nameserver.go
@@ -168,7 +168,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 		}
 
 	default:
-		tracer.Warningf("nameserver: external request for %s%s, ignoring", q.FQDN, q.QType)
+		tracer.Warningf("nameserver: external request from %s for %s%s, ignoring", remoteAddr, q.FQDN, q.QType)
 		return reply(nsutil.Refused("external queries are not permitted"))
 	}
 	conn.Lock()

From 42eb3a1d0e322d035cc0d8fe37f42eeb22cf99d2 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 15 Apr 2022 13:07:13 +0200
Subject: [PATCH 21/86] Use more verbose names for iptables chains

---
 firewall/interception/nfqueue_linux.go | 110 +++++++++++++------------
 1 file changed, 58 insertions(+), 52 deletions(-)

diff --git a/firewall/interception/nfqueue_linux.go b/firewall/interception/nfqueue_linux.go
index 0b207e41..f117cd67 100644
--- a/firewall/interception/nfqueue_linux.go
+++ b/firewall/interception/nfqueue_linux.go
@@ -46,83 +46,89 @@ type nfQueue interface {
 
 func init() {
 	v4chains = []string{
-		"mangle C170",
-		"mangle C171",
-		"filter C17",
+		"mangle PORTMASTER-INGEST-OUTPUT",
+		"mangle PORTMASTER-INGEST-INPUT",
+		"filter PORTMASTER-FILTER",
+		"nat PORTMASTER-REDIRECT",
 	}
 
 	v4rules = []string{
-		"mangle C170 -j CONNMARK --restore-mark",
-		"mangle C170 -m mark --mark 0 -j NFQUEUE --queue-num 17040 --queue-bypass",
+		"mangle PORTMASTER-INGEST-OUTPUT -j CONNMARK --restore-mark",
+		"mangle PORTMASTER-INGEST-OUTPUT -m mark --mark 0 -j NFQUEUE --queue-num 17040 --queue-bypass",
 
-		"mangle C171 -j CONNMARK --restore-mark",
-		"mangle C171 -m mark --mark 0 -j NFQUEUE --queue-num 17140 --queue-bypass",
+		"mangle PORTMASTER-INGEST-INPUT -j CONNMARK --restore-mark",
+		"mangle PORTMASTER-INGEST-INPUT -m mark --mark 0 -j NFQUEUE --queue-num 17140 --queue-bypass",
 
-		"filter C17 -m mark --mark 0 -j DROP",
-		"filter C17 -m mark --mark 1700 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 0 -j DROP",
+		"filter PORTMASTER-FILTER -m mark --mark 1700 -j RETURN",
 		// Accepting ICMP packets with mark 1701 is required for rejecting to work,
 		// as the rejection ICMP packet will have the same mark. Blocked ICMP
 		// packets will always result in a drop within the Portmaster.
-		"filter C17 -m mark --mark 1701 -p icmp -j RETURN",
-		"filter C17 -m mark --mark 1701 -j REJECT --reject-with icmp-host-prohibited",
-		"filter C17 -m mark --mark 1702 -j DROP",
-		"filter C17 -j CONNMARK --save-mark",
-		"filter C17 -m mark --mark 1710 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1701 -p icmp -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1701 -j REJECT --reject-with icmp-host-prohibited",
+		"filter PORTMASTER-FILTER -m mark --mark 1702 -j DROP",
+		"filter PORTMASTER-FILTER -j CONNMARK --save-mark",
+		"filter PORTMASTER-FILTER -m mark --mark 1710 -j RETURN",
 		// Accepting ICMP packets with mark 1711 is required for rejecting to work,
 		// as the rejection ICMP packet will have the same mark. Blocked ICMP
 		// packets will always result in a drop within the Portmaster.
-		"filter C17 -m mark --mark 1711 -p icmp -j RETURN",
-		"filter C17 -m mark --mark 1711 -j REJECT --reject-with icmp-host-prohibited",
-		"filter C17 -m mark --mark 1712 -j DROP",
-		"filter C17 -m mark --mark 1717 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1711 -p icmp -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1711 -j REJECT --reject-with icmp-host-prohibited",
+		"filter PORTMASTER-FILTER -m mark --mark 1712 -j DROP",
+		"filter PORTMASTER-FILTER -m mark --mark 1717 -j RETURN",
+
+		"nat PORTMASTER-REDIRECT -m mark --mark 1799 -p udp -j DNAT --to 127.0.0.17:53",
+		"nat PORTMASTER-REDIRECT -m mark --mark 1717 -p tcp -j DNAT --to 127.0.0.17:717",
+		"nat PORTMASTER-REDIRECT -m mark --mark 1717 -p udp -j DNAT --to 127.0.0.17:717",
+		// "nat PORTMASTER-REDIRECT -m mark --mark 1717 ! -p tcp ! -p udp -j DNAT --to 127.0.0.17",
 	}
 
 	v4once = []string{
-		"mangle OUTPUT -j C170",
-		"mangle INPUT -j C171",
-		"filter OUTPUT -j C17",
-		"filter INPUT -j C17",
-		"nat OUTPUT -m mark --mark 1799 -p udp -j DNAT --to 127.0.0.17:53",
-		"nat OUTPUT -m mark --mark 1717 -p tcp -j DNAT --to 127.0.0.17:717",
-		"nat OUTPUT -m mark --mark 1717 -p udp -j DNAT --to 127.0.0.17:717",
-		// "nat OUTPUT -m mark --mark 1717 ! -p tcp ! -p udp -j DNAT --to 127.0.0.17",
+		"mangle OUTPUT -j PORTMASTER-INGEST-OUTPUT",
+		"mangle INPUT -j PORTMASTER-INGEST-INPUT",
+		"filter OUTPUT -j PORTMASTER-FILTER",
+		"filter INPUT -j PORTMASTER-FILTER",
+		"nat OUTPUT -j PORTMASTER-REDIRECT",
 	}
 
 	v6chains = []string{
-		"mangle C170",
-		"mangle C171",
-		"filter C17",
+		"mangle PORTMASTER-INGEST-OUTPUT",
+		"mangle PORTMASTER-INGEST-INPUT",
+		"filter PORTMASTER-FILTER",
+		"nat PORTMASTER-REDIRECT",
 	}
 
 	v6rules = []string{
-		"mangle C170 -j CONNMARK --restore-mark",
-		"mangle C170 -m mark --mark 0 -j NFQUEUE --queue-num 17060 --queue-bypass",
+		"mangle PORTMASTER-INGEST-OUTPUT -j CONNMARK --restore-mark",
+		"mangle PORTMASTER-INGEST-OUTPUT -m mark --mark 0 -j NFQUEUE --queue-num 17060 --queue-bypass",
 
-		"mangle C171 -j CONNMARK --restore-mark",
-		"mangle C171 -m mark --mark 0 -j NFQUEUE --queue-num 17160 --queue-bypass",
+		"mangle PORTMASTER-INGEST-INPUT -j CONNMARK --restore-mark",
+		"mangle PORTMASTER-INGEST-INPUT -m mark --mark 0 -j NFQUEUE --queue-num 17160 --queue-bypass",
 
-		"filter C17 -m mark --mark 0 -j DROP",
-		"filter C17 -m mark --mark 1700 -j RETURN",
-		"filter C17 -m mark --mark 1701 -p icmpv6 -j RETURN",
-		"filter C17 -m mark --mark 1701 -j REJECT --reject-with icmp6-adm-prohibited",
-		"filter C17 -m mark --mark 1702 -j DROP",
-		"filter C17 -j CONNMARK --save-mark",
-		"filter C17 -m mark --mark 1710 -j RETURN",
-		"filter C17 -m mark --mark 1711 -p icmpv6 -j RETURN",
-		"filter C17 -m mark --mark 1711 -j REJECT --reject-with icmp6-adm-prohibited",
-		"filter C17 -m mark --mark 1712 -j DROP",
-		"filter C17 -m mark --mark 1717 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 0 -j DROP",
+		"filter PORTMASTER-FILTER -m mark --mark 1700 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1701 -p icmpv6 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1701 -j REJECT --reject-with icmp6-adm-prohibited",
+		"filter PORTMASTER-FILTER -m mark --mark 1702 -j DROP",
+		"filter PORTMASTER-FILTER -j CONNMARK --save-mark",
+		"filter PORTMASTER-FILTER -m mark --mark 1710 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1711 -p icmpv6 -j RETURN",
+		"filter PORTMASTER-FILTER -m mark --mark 1711 -j REJECT --reject-with icmp6-adm-prohibited",
+		"filter PORTMASTER-FILTER -m mark --mark 1712 -j DROP",
+		"filter PORTMASTER-FILTER -m mark --mark 1717 -j RETURN",
+
+		"nat PORTMASTER-REDIRECT -m mark --mark 1799 -p udp -j DNAT --to [::1]:53",
+		"nat PORTMASTER-REDIRECT -m mark --mark 1717 -p tcp -j DNAT --to [::1]:717",
+		"nat PORTMASTER-REDIRECT -m mark --mark 1717 -p udp -j DNAT --to [::1]:717",
+		// "nat PORTMASTER-REDIRECT -m mark --mark 1717 ! -p tcp ! -p udp -j DNAT --to [::1]",
 	}
 
 	v6once = []string{
-		"mangle OUTPUT -j C170",
-		"mangle INPUT -j C171",
-		"filter OUTPUT -j C17",
-		"filter INPUT -j C17",
-		"nat OUTPUT -m mark --mark 1799 -p udp -j DNAT --to [::1]:53",
-		"nat OUTPUT -m mark --mark 1717 -p tcp -j DNAT --to [::1]:717",
-		"nat OUTPUT -m mark --mark 1717 -p udp -j DNAT --to [::1]:717",
-		// "nat OUTPUT -m mark --mark 1717 ! -p tcp ! -p udp -j DNAT --to [::1]",
+		"mangle OUTPUT -j PORTMASTER-INGEST-OUTPUT",
+		"mangle INPUT -j PORTMASTER-INGEST-INPUT",
+		"filter OUTPUT -j PORTMASTER-FILTER",
+		"filter INPUT -j PORTMASTER-FILTER",
+		"nat OUTPUT -j PORTMASTER-REDIRECT",
 	}
 
 	// Reverse because we'd like to insert in a loop

From 9a39caf22bc37b29581370de3d62cbd5f1f1c953 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 26 Apr 2022 15:02:05 +0200
Subject: [PATCH 22/86] Resolve to real file paths before checking path based
 API access

---
 firewall/api.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/firewall/api.go b/firewall/api.go
index 18c675e9..99ae36b5 100644
--- a/firewall/api.go
+++ b/firewall/api.go
@@ -22,11 +22,7 @@ import (
 )
 
 const (
-	deniedMsgUnidentified = `%wFailed to identify the requesting process.
-You can enable the Development Mode to disable API authentication for development purposes.
-
-If you are seeing this message in the Portmaster App, please restart the app or right-click and select "Reload".
-In the future, this issue will be remediated automatically.`
+	deniedMsgUnidentified = `%wFailed to identify the requesting process. Reload to try again.`
 
 	deniedMsgSystem = `%wSystem access to the Portmaster API is not permitted.
 You can enable the Development Mode to disable API authentication for development purposes.`
@@ -136,6 +132,12 @@ func authenticateAPIRequest(ctx context.Context, pktInfo *packet.Info) (retry bo
 	if authenticatedPath == "" {
 		return false, fmt.Errorf(deniedMsgMisconfigured, api.ErrAPIAccessDeniedMessage) //nolint:stylecheck // message for user
 	}
+	// Get real path.
+	authenticatedPath, err = filepath.EvalSymlinks(authenticatedPath)
+	if err != nil {
+		return false, fmt.Errorf(deniedMsgUnidentified, api.ErrAPIAccessDeniedMessage) //nolint:stylecheck // message for user
+	}
+	// Add filepath separator to confine to directory.
 	authenticatedPath += string(filepath.Separator)
 
 	// Get process of request.
@@ -157,8 +159,10 @@ func authenticateAPIRequest(ctx context.Context, pktInfo *packet.Info) (retry bo
 				break checkLevelsLoop
 			default: // normal process
 				// Check if the requesting process is in database root / updates dir.
-				if strings.HasPrefix(proc.Path, authenticatedPath) {
-					return false, nil
+				if realPath, err := filepath.EvalSymlinks(proc.Path); err == nil {
+					if strings.HasPrefix(realPath, authenticatedPath) {
+						return false, nil
+					}
 				}
 			}
 

From d0e3107ea52af81ba7a8388fe61300e71f5f9a90 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 26 Apr 2022 15:02:39 +0200
Subject: [PATCH 23/86] Only warn about unexpected parent process when not in
 dev mode

---
 updates/upgrader.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/updates/upgrader.go b/updates/upgrader.go
index aabc2441..ebab6870 100644
--- a/updates/upgrader.go
+++ b/updates/upgrader.go
@@ -230,7 +230,10 @@ func warnOnIncorrectParentPath() {
 		return
 	}
 	if parentName != expectedFileName {
-		log.Warningf("updates: parent process does not seem to be portmaster-start, name is %s", parentName)
+		// Only warn about this if not in dev mode.
+		if !devMode() {
+			log.Warningf("updates: parent process does not seem to be portmaster-start, name is %s", parentName)
+		}
 
 		// TODO(ppacher): once we released a new installer and folks had time
 		//                to update we should send a module warning/hint to the

From 998662d928523dfaf621c43d83cc256579819ee6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 26 Apr 2022 15:03:01 +0200
Subject: [PATCH 24/86] Add MatchMulti function to match endpoints list against
 multiple entities

---
 profile/endpoints/endpoints.go | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/profile/endpoints/endpoints.go b/profile/endpoints/endpoints.go
index 05273f95..6ed3ad04 100644
--- a/profile/endpoints/endpoints.go
+++ b/profile/endpoints/endpoints.go
@@ -90,7 +90,32 @@ func (e Endpoints) IsSet() bool {
 // Match checks whether the given entity matches any of the endpoint definitions in the list.
 func (e Endpoints) Match(ctx context.Context, entity *intel.Entity) (result EPResult, reason Reason) {
 	for _, entry := range e {
-		if entry != nil {
+		if entry == nil {
+			continue
+		}
+
+		if result, reason = entry.Matches(ctx, entity); result != NoMatch {
+			return
+		}
+	}
+
+	return NoMatch, nil
+}
+
+// MatchMulti checks whether the given entities match any of the endpoint
+// definitions in the list. Every rule is evaluated against all given entities
+// and only if not match was registered, the next rule is evaluated.
+func (e Endpoints) MatchMulti(ctx context.Context, entities ...*intel.Entity) (result EPResult, reason Reason) {
+	for _, entry := range e {
+		if entry == nil {
+			continue
+		}
+
+		for _, entity := range entities {
+			if entity == nil {
+				continue
+			}
+
 			if result, reason = entry.Matches(ctx, entity); result != NoMatch {
 				return
 			}

From 96e7486257494402066239ad6bbfa2876cf61b70 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 5 May 2022 11:08:05 +0200
Subject: [PATCH 25/86] Update deps

---
 go.mod | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 go.sum | 23 +++++++++++++---------
 2 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/go.mod b/go.mod
index b128b856..715b49dc 100644
--- a/go.mod
+++ b/go.mod
@@ -6,14 +6,14 @@ require (
 	github.com/agext/levenshtein v1.2.3
 	github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d
 	github.com/coreos/go-iptables v0.6.0
-	github.com/florianl/go-nfqueue v1.3.0
+	github.com/florianl/go-nfqueue v1.3.1
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.4.0
 	github.com/miekg/dns v1.1.48
 	github.com/oschwald/maxminddb-golang v1.9.0
-	github.com/safing/portbase v0.14.2
+	github.com/safing/portbase v0.14.3
 	github.com/safing/spn v0.4.6
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
@@ -21,7 +21,60 @@ require (
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220412020605-290c469a71a5
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6
+)
+
+require (
+	github.com/VictoriaMetrics/metrics v1.18.1 // indirect
+	github.com/aead/ecdh v0.2.0 // indirect
+	github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 // indirect
+	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/native v1.0.0 // indirect
+	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/mdlayher/netlink v1.6.0 // indirect
+	github.com/mdlayher/socket v0.2.3 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rot256/pblind v0.0.0-20211117203330-22455f90b565 // indirect
+	github.com/safing/jess v0.2.3 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/seehuhn/fortuna v1.0.1 // indirect
+	github.com/seehuhn/sha256d v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.14.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.4 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/valyala/fastrand v1.1.0 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
diff --git a/go.sum b/go.sum
index e249db8a..36c059c6 100644
--- a/go.sum
+++ b/go.sum
@@ -263,7 +263,6 @@ github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70d
 github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
 github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
@@ -298,8 +297,9 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/florianl/go-nfqueue v1.3.0 h1:cvZGUM6k1zxkokHM79Hg/q39cVjf3WAQZ/46ncpuhkc=
 github.com/florianl/go-nfqueue v1.3.0/go.mod h1:sA7IQtpB3zxpdwJ4y4999SjK+1lx91TEqBBB4CIlFX0=
+github.com/florianl/go-nfqueue v1.3.1 h1:khQ9fYCrjbu5CF8dZF55G2RTIEIQRI0Aj5k3msJR6Gw=
+github.com/florianl/go-nfqueue v1.3.1/go.mod h1:aHWbgkhryJxF5XxYvJ3oRZpdD4JP74Zu/hP1zuhja+M=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -354,7 +354,6 @@ github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3a
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.5/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -424,8 +423,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
 github.com/google/go-replayers/httpreplay v1.0.0/go.mod h1:LJhKoTwS5Wy5Ld/peq8dFFG5OfJyHEz7ft+DsTUv25M=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -815,8 +815,9 @@ github.com/safing/portbase v0.13.5/go.mod h1:5vj5IK5WJoSGareDe6yCMZfnF7txVRx7jZy
 github.com/safing/portbase v0.13.6/go.mod h1:G0maDSQxYDuluNhMzA1zVd/nfXawfECv5H7+fnTfVhM=
 github.com/safing/portbase v0.14.0/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.1/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
-github.com/safing/portbase v0.14.2 h1:xL1TebfvWsSbf5AsZxdY0+/+mArgkqmSnmnqsRZpZYw=
 github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.3 h1:ig4gwFwWheMTmzCsnZCW1+Hcz+QQaJwHEXTzJUS+drc=
+github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=
@@ -937,8 +938,9 @@ github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl
 github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
+github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/match v1.1.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -1076,8 +1078,9 @@ golang.org/x/crypto v0.0.0-20220213190939-1e6e3497d506/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1195,8 +1198,9 @@ golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1351,8 +1355,9 @@ golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

From 58a66dca4ebada3b747ed93bd0ce105b5edbf7bc Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 5 May 2022 11:13:27 +0200
Subject: [PATCH 26/86] Update SPN

---
 go.mod | 2 +-
 go.sum | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 715b49dc..6bc6431d 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.48
 	github.com/oschwald/maxminddb-golang v1.9.0
 	github.com/safing/portbase v0.14.3
-	github.com/safing/spn v0.4.6
+	github.com/safing/spn v0.4.7
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
diff --git a/go.sum b/go.sum
index 36c059c6..1d6c3c02 100644
--- a/go.sum
+++ b/go.sum
@@ -833,6 +833,7 @@ github.com/safing/portmaster v0.8.0/go.mod h1:lY2/WvOlH8kl1AwkixdWCjlo+PZQv+oEOQ
 github.com/safing/portmaster v0.8.5-interdep/go.mod h1:A+zAVEKjr057ktgiMSJRdUmOF+FPW8XY/5LqGnbsKbU=
 github.com/safing/portmaster v0.8.5/go.mod h1:MqOlFwHcIx/109Ugutz/CG23znuuXCRVHcIcfX0VC/c=
 github.com/safing/portmaster v0.8.7/go.mod h1:RUgCWt5v22jDUOtJfOwApi//Kt8RTZQhlREcBc+L4z8=
+github.com/safing/portmaster v0.8.9-interdep/go.mod h1:1hK7QpvFVlb/sglkc3SKj+RXMGBuk0wqO2s3pvMg1Xs=
 github.com/safing/spn v0.3.4/go.mod h1:TfzNsZCbnlWv0UFDILFOUSudVKJZlnBVoR1fDXrjOK0=
 github.com/safing/spn v0.3.5/go.mod h1:jHkFF2Yu1fnjFu4KXjVA+iagMr/z4eB4p3jiwikvKj8=
 github.com/safing/spn v0.3.6/go.mod h1:RSeFb/h5Wt3yDVezXj3lhXJ/Iwd7FbtsGf5E+p5J2YQ=
@@ -850,6 +851,8 @@ github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
 github.com/safing/spn v0.4.6 h1:jOxFdVpSSXCCCiNktBzPNfom6Tf1qtsjr2USFiTGHIM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
+github.com/safing/spn v0.4.7 h1:YCensNJ/6nW9TzQO+lv2i/58pcIDillWYgAclWxR7UA=
+github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

From becb6b373f98dd95b758f01eb5ca73069bb9942f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 5 May 2022 11:13:34 +0200
Subject: [PATCH 27/86] Bump core version

---
 cmds/portmaster-core/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index 27dd5175..ff2888bf 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.8", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.9", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")

From 7c5a6110fcde785d8354f92c3ce0d18aae106bbd Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 10 May 2022 16:01:07 +0200
Subject: [PATCH 28/86] Update Go workflow

---
 .github/workflows/go.yml | 57 ++++++++--------------------------------
 1 file changed, 11 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 1b14cf42..e5d1ca0b 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -16,31 +16,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v3
       with:
-        go-version: '^1.15'
-
-    # nektos/act does not have sudo install but we need it on GH actions so
-    # try to install it.
-    - name: Install sudo
-      run: bash -c "apt-get update || true ; apt-get install sudo || true"
-      env:
-        DEBIAN_FRONTEND: noninteractive 
-
-    - name: Install git and gcc
-      run: sudo bash -c "apt-get update && apt-get install -y git gcc libc6-dev"
-      env:
-        DEBIAN_FRONTEND: noninteractive 
+        go-version: '^1.18'
 
     - name: Run golangci-lint
-      uses: golangci/golangci-lint-action@v2
+      uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.44
+        version: v1.45.1
         only-new-issues: true
         args: -c ./.golangci.yml
-        skip-go-installation: true
 
     - name: Get dependencies
       run: go mod download
@@ -48,42 +35,20 @@ jobs:
     - name: Run go vet
       run: go vet ./...
 
-    # golint is run (sufficiently; with excludes) as a part of golangci-lint.
-    # - name: Install golint
-    #   run: bash -c "GOBIN=$(pwd) go get -u golang.org/x/lint/golint"
-    #
-    # - name: Run golint
-    #   run: ./golint -set_exit_status -min_confidence 1.0 ./...
-   
-    # gofmt is run (sufficiently; with excludes) as a part of golangci-lint.
-    # - name: Run gofmt
-    #   run: bash -c 'test -z "$(gofmt -s -l .)"'
-
   test:
     name: Test
     runs-on: ubuntu-latest
     steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+    - name: Check out code
+      uses: actions/checkout@v3
 
-    - uses: actions/setup-go@v2
+    - name: Setup Go
+      uses: actions/setup-go@v3
       with:
-        go-version: '^1.15'
-
-    # nektos/act does not have sudo install but we need it on GH actions so
-    # try to install it.
-    - name: Install sudo
-      run: bash -c "apt-get update || true ; apt-get install sudo || true"
-      env:
-        DEBIAN_FRONTEND: noninteractive 
-
-    - name: Install git and gcc
-      run: sudo bash -c "apt-get update && apt-get install -y git gcc libc6-dev"
-      env:
-        DEBIAN_FRONTEND: noninteractive 
+        go-version: '^1.18'
 
     - name: Get dependencies
       run: go mod download
 
-    - name: Test
+    - name: Run tests
       run: ./test --test-only

From ec6a3f03e8c855c623c9ef2af75c9bb4409ee25a Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 11 May 2022 13:19:31 +0200
Subject: [PATCH 29/86] Fix shutting down nfq interception with go-nfqueue
 v1.3.1

---
 firewall/interception/nfq/nfq.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/firewall/interception/nfq/nfq.go b/firewall/interception/nfq/nfq.go
index 820b7847..1128c467 100644
--- a/firewall/interception/nfq/nfq.go
+++ b/firewall/interception/nfq/nfq.go
@@ -25,6 +25,7 @@ type Queue struct {
 	nf                   atomic.Value
 	packets              chan pmpacket.Packet
 	cancelSocketCallback context.CancelFunc
+	canceled             *abool.AtomicBool
 	restart              chan struct{}
 
 	pendingVerdicts  uint64
@@ -50,6 +51,7 @@ func New(qid uint16, v6 bool) (*Queue, error) { //nolint:gocognit
 		restart:              make(chan struct{}, 1),
 		packets:              make(chan pmpacket.Packet, 1000),
 		cancelSocketCallback: cancel,
+		canceled:             abool.New(),
 		verdictCompleted:     make(chan struct{}, 1),
 	}
 
@@ -142,6 +144,11 @@ func (q *Queue) handleError(e error) int {
 		}
 	}
 
+	// Check if we have closed the queue already.
+	if q.canceled.IsSet() {
+		return 1
+	}
+
 	// Check if the queue was already closed. Unfortunately, the exposed error
 	// variable is in an internal stdlib package. Therefore, check for the error
 	// string instead. :(
@@ -222,6 +229,7 @@ func (q *Queue) Destroy() {
 		return
 	}
 
+	q.canceled.Set()
 	q.cancelSocketCallback()
 
 	if nf := q.getNfq(); nf != nil {

From beefc836f6a24fc6073c77cbd41c534370b88ab9 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 11 May 2022 13:22:25 +0200
Subject: [PATCH 30/86] Increase supported size of windows network state table
 entries

---
 network/iphelper/get.go         |  2 +-
 network/iphelper/iphelper.go    |  2 +-
 network/iphelper/tables.go      | 52 +++++++++++++++++++++++++++------
 network/iphelper/tables_test.go |  2 +-
 4 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/network/iphelper/get.go b/network/iphelper/get.go
index e92f929c..31f1c925 100644
--- a/network/iphelper/get.go
+++ b/network/iphelper/get.go
@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows
 
 package iphelper
 
diff --git a/network/iphelper/iphelper.go b/network/iphelper/iphelper.go
index 5498879a..5fa1ea12 100644
--- a/network/iphelper/iphelper.go
+++ b/network/iphelper/iphelper.go
@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows
 
 package iphelper
 
diff --git a/network/iphelper/tables.go b/network/iphelper/tables.go
index 9d59f7d7..94998d7e 100644
--- a/network/iphelper/tables.go
+++ b/network/iphelper/tables.go
@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows
 
 package iphelper
 
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"unsafe"
 
+	"github.com/safing/portbase/log"
 	"github.com/safing/portmaster/network/socket"
 
 	"golang.org/x/sys/windows"
@@ -28,7 +29,7 @@ const (
 type iphelperTCPTable struct {
 	// docs: https://msdn.microsoft.com/en-us/library/windows/desktop/aa366921(v=vs.85).aspx
 	numEntries uint32
-	table      [4096]iphelperTCPRow
+	table      [maxStateTableEntries]iphelperTCPRow
 }
 
 type iphelperTCPRow struct {
@@ -44,7 +45,7 @@ type iphelperTCPRow struct {
 type iphelperTCP6Table struct {
 	// docs: https://msdn.microsoft.com/en-us/library/windows/desktop/aa366905(v=vs.85).aspx
 	numEntries uint32
-	table      [4096]iphelperTCP6Row
+	table      [maxStateTableEntries]iphelperTCP6Row
 }
 
 type iphelperTCP6Row struct {
@@ -62,7 +63,7 @@ type iphelperTCP6Row struct {
 type iphelperUDPTable struct {
 	// docs: https://msdn.microsoft.com/en-us/library/windows/desktop/aa366932(v=vs.85).aspx
 	numEntries uint32
-	table      [4096]iphelperUDPRow
+	table      [maxStateTableEntries]iphelperUDPRow
 }
 
 type iphelperUDPRow struct {
@@ -75,7 +76,7 @@ type iphelperUDPRow struct {
 type iphelperUDP6Table struct {
 	// docs: https://msdn.microsoft.com/en-us/library/windows/desktop/aa366925(v=vs.85).aspx
 	numEntries uint32
-	table      [4096]iphelperUDP6Row
+	table      [maxStateTableEntries]iphelperUDP6Row
 }
 
 type iphelperUDP6Row struct {
@@ -125,6 +126,11 @@ const (
 	// maxBufSize is the maximum size we will allocate for responses. This was
 	// previously set at 65k, which was too little for some production cases.
 	maxBufSize = 1048576 // 2^20B, 1MB
+
+	// maxStateTableEntries is the maximum supported amount of entries of the
+	// state tables.
+	// This is never allocated, but just casted to from an unsafe pointer.
+	maxStateTableEntries = 65535
 )
 
 var (
@@ -261,7 +267,14 @@ func (ipHelper *IPHelper) getTable(ipVersion, protocol uint8) (connections []*so
 	case protocol == TCP && ipVersion == IPv4:
 
 		tcpTable := (*iphelperTCPTable)(unsafe.Pointer(&buf[0]))
-		table := tcpTable.table[:tcpTable.numEntries]
+		// Check if we got more entries than supported.
+		tableEntries := tcpTable.numEntries
+		if tableEntries > maxStateTableEntries {
+			tableEntries = maxStateTableEntries
+			log.Warningf("network/iphelper: received TCPv4 table with more entries than supported: %d/%d", tcpTable.numEntries, maxStateTableEntries)
+		}
+		// Cap table to actual entries.
+		table := tcpTable.table[:tableEntries]
 
 		for _, row := range table {
 			if row.state == iphelperTCPStateListen {
@@ -290,7 +303,14 @@ func (ipHelper *IPHelper) getTable(ipVersion, protocol uint8) (connections []*so
 	case protocol == TCP && ipVersion == IPv6:
 
 		tcpTable := (*iphelperTCP6Table)(unsafe.Pointer(&buf[0]))
-		table := tcpTable.table[:tcpTable.numEntries]
+		// Check if we got more entries than supported.
+		tableEntries := tcpTable.numEntries
+		if tableEntries > maxStateTableEntries {
+			tableEntries = maxStateTableEntries
+			log.Warningf("network/iphelper: received TCPv6 table with more entries than supported: %d/%d", tcpTable.numEntries, maxStateTableEntries)
+		}
+		// Cap table to actual entries.
+		table := tcpTable.table[:tableEntries]
 
 		for _, row := range table {
 			if row.state == iphelperTCPStateListen {
@@ -319,7 +339,14 @@ func (ipHelper *IPHelper) getTable(ipVersion, protocol uint8) (connections []*so
 	case protocol == UDP && ipVersion == IPv4:
 
 		udpTable := (*iphelperUDPTable)(unsafe.Pointer(&buf[0]))
-		table := udpTable.table[:udpTable.numEntries]
+		// Check if we got more entries than supported.
+		tableEntries := udpTable.numEntries
+		if tableEntries > maxStateTableEntries {
+			tableEntries = maxStateTableEntries
+			log.Warningf("network/iphelper: received UDPv4 table with more entries than supported: %d/%d", udpTable.numEntries, maxStateTableEntries)
+		}
+		// Cap table to actual entries.
+		table := udpTable.table[:tableEntries]
 
 		for _, row := range table {
 			binds = append(binds, &socket.BindInfo{
@@ -334,7 +361,14 @@ func (ipHelper *IPHelper) getTable(ipVersion, protocol uint8) (connections []*so
 	case protocol == UDP && ipVersion == IPv6:
 
 		udpTable := (*iphelperUDP6Table)(unsafe.Pointer(&buf[0]))
-		table := udpTable.table[:udpTable.numEntries]
+		// Check if we got more entries than supported.
+		tableEntries := udpTable.numEntries
+		if tableEntries > maxStateTableEntries {
+			tableEntries = maxStateTableEntries
+			log.Warningf("network/iphelper: received UDPv6 table with more entries than supported: %d/%d", udpTable.numEntries, maxStateTableEntries)
+		}
+		// Cap table to actual entries.
+		table := udpTable.table[:tableEntries]
 
 		for _, row := range table {
 			binds = append(binds, &socket.BindInfo{
diff --git a/network/iphelper/tables_test.go b/network/iphelper/tables_test.go
index e996219e..9b8c48f5 100644
--- a/network/iphelper/tables_test.go
+++ b/network/iphelper/tables_test.go
@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows
 
 package iphelper
 

From 217901c179e9bf7af251b51d25684a8d3728ed7f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Mon, 16 May 2022 09:17:29 +0200
Subject: [PATCH 31/86] Update deps and SPN

---
 go.mod | 16 ++++++++--------
 go.sum | 29 ++++++++++++++++++++---------
 2 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/go.mod b/go.mod
index 6bc6431d..6fa2cb23 100644
--- a/go.mod
+++ b/go.mod
@@ -11,19 +11,19 @@ require (
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.4.0
-	github.com/miekg/dns v1.1.48
+	github.com/miekg/dns v1.1.49
 	github.com/oschwald/maxminddb-golang v1.9.0
-	github.com/safing/portbase v0.14.3
-	github.com/safing/spn v0.4.7
+	github.com/safing/portbase v0.14.4
+	github.com/safing/spn v0.4.9
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6
+	golang.org/x/net v0.0.0-20220513224357-95641704303c
+	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
+	golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a
 )
 
 require (
@@ -63,7 +63,7 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.4 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
-	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/tklauser/numcpus v0.5.0 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
@@ -71,7 +71,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
diff --git a/go.sum b/go.sum
index 1d6c3c02..fa786ada 100644
--- a/go.sum
+++ b/go.sum
@@ -687,8 +687,9 @@ github.com/miekg/dns v1.1.44/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7Xn
 github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
-github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
 github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
+github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -816,8 +817,9 @@ github.com/safing/portbase v0.13.6/go.mod h1:G0maDSQxYDuluNhMzA1zVd/nfXawfECv5H7
 github.com/safing/portbase v0.14.0/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.1/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
-github.com/safing/portbase v0.14.3 h1:ig4gwFwWheMTmzCsnZCW1+Hcz+QQaJwHEXTzJUS+drc=
 github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.4 h1:yEWlvh2w0gVdjrA9oo5QPFV5oluJ6YDoKRPtmFb9B4c=
+github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=
@@ -834,6 +836,7 @@ github.com/safing/portmaster v0.8.5-interdep/go.mod h1:A+zAVEKjr057ktgiMSJRdUmOF
 github.com/safing/portmaster v0.8.5/go.mod h1:MqOlFwHcIx/109Ugutz/CG23znuuXCRVHcIcfX0VC/c=
 github.com/safing/portmaster v0.8.7/go.mod h1:RUgCWt5v22jDUOtJfOwApi//Kt8RTZQhlREcBc+L4z8=
 github.com/safing/portmaster v0.8.9-interdep/go.mod h1:1hK7QpvFVlb/sglkc3SKj+RXMGBuk0wqO2s3pvMg1Xs=
+github.com/safing/portmaster v0.8.9/go.mod h1:tv0rxO76hrpBLdArN7YTypOaseH6zgQ2gLI2zCknk9Q=
 github.com/safing/spn v0.3.4/go.mod h1:TfzNsZCbnlWv0UFDILFOUSudVKJZlnBVoR1fDXrjOK0=
 github.com/safing/spn v0.3.5/go.mod h1:jHkFF2Yu1fnjFu4KXjVA+iagMr/z4eB4p3jiwikvKj8=
 github.com/safing/spn v0.3.6/go.mod h1:RSeFb/h5Wt3yDVezXj3lhXJ/Iwd7FbtsGf5E+p5J2YQ=
@@ -849,10 +852,12 @@ github.com/safing/spn v0.4.0/go.mod h1:0jBetnYCfxqO5PJskhPOxJ/v6VRfE+bQU98XW240B
 github.com/safing/spn v0.4.2/go.mod h1:yZPezHDEYyhei8n13tTxjQCGq6LRr5svz9WFAAeDPec=
 github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW2M=
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
-github.com/safing/spn v0.4.6 h1:jOxFdVpSSXCCCiNktBzPNfom6Tf1qtsjr2USFiTGHIM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
-github.com/safing/spn v0.4.7 h1:YCensNJ/6nW9TzQO+lv2i/58pcIDillWYgAclWxR7UA=
 github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
+github.com/safing/spn v0.4.8 h1:C5QOl03hypDaC4qvFtWjAKmSJIOjfqYVGT+sTnUIO/E=
+github.com/safing/spn v0.4.8/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
+github.com/safing/spn v0.4.9 h1:p+TRS2y1yAY4CYyHiVuy+V3NcA9p826NtYksuBaSl64=
+github.com/safing/spn v0.4.9/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -965,8 +970,9 @@ github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03O
 github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
 github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
-github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
 github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
+github.com/tklauser/numcpus v0.5.0 h1:ooe7gN0fg6myJ0EKoTAf5hebTZrH52px3New/D9iJ+A=
+github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKoflh+RQjo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -1082,8 +1088,10 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c=
+golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1202,8 +1210,9 @@ golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220513224357-95641704303c h1:nF9mHSvoKBLkQNQhJZNsc66z2UzAMUbLGjC95CF3pU0=
+golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1234,8 +1243,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
+golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1359,8 +1369,9 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXHCguynGhKjWVsIUt5cY=
+golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

From e953bc7f3ff48267f2860ea1129ebbf215004d71 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Mon, 16 May 2022 09:17:46 +0200
Subject: [PATCH 32/86] Bump core version

---
 cmds/portmaster-core/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index ff2888bf..2b113ab3 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.9", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.10", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")

From 932f6002b472af381ed5044ca1ca210124effc33 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 18 May 2022 09:34:27 +0200
Subject: [PATCH 33/86] Use alternative approach to fix nfqueue closing process

This commit reverts commit ec6a3f03e8c855c623c9ef2af75c9bb4409ee25a.
---
 firewall/interception/nfq/nfq.go | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/firewall/interception/nfq/nfq.go b/firewall/interception/nfq/nfq.go
index 1128c467..f9a688ff 100644
--- a/firewall/interception/nfq/nfq.go
+++ b/firewall/interception/nfq/nfq.go
@@ -25,7 +25,6 @@ type Queue struct {
 	nf                   atomic.Value
 	packets              chan pmpacket.Packet
 	cancelSocketCallback context.CancelFunc
-	canceled             *abool.AtomicBool
 	restart              chan struct{}
 
 	pendingVerdicts  uint64
@@ -51,7 +50,6 @@ func New(qid uint16, v6 bool) (*Queue, error) { //nolint:gocognit
 		restart:              make(chan struct{}, 1),
 		packets:              make(chan pmpacket.Packet, 1000),
 		cancelSocketCallback: cancel,
-		canceled:             abool.New(),
 		verdictCompleted:     make(chan struct{}, 1),
 	}
 
@@ -144,11 +142,6 @@ func (q *Queue) handleError(e error) int {
 		}
 	}
 
-	// Check if we have closed the queue already.
-	if q.canceled.IsSet() {
-		return 1
-	}
-
 	// Check if the queue was already closed. Unfortunately, the exposed error
 	// variable is in an internal stdlib package. Therefore, check for the error
 	// string instead. :(
@@ -160,7 +153,8 @@ func (q *Queue) handleError(e error) int {
 
 	// Close the existing socket
 	if nf := q.getNfq(); nf != nil {
-		_ = nf.Close()
+		// Call Close() on the Con directly, as nf.Close() calls waitgroup.Wait(), which then may deadlock.
+		_ = nf.Con.Close()
 	}
 
 	// Trigger a restart of the queue
@@ -229,7 +223,6 @@ func (q *Queue) Destroy() {
 		return
 	}
 
-	q.canceled.Set()
 	q.cancelSocketCallback()
 
 	if nf := q.getNfq(); nf != nil {

From 3772a587c17de49bdcad71593e0384e880d37256 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 18 May 2022 13:26:46 +0200
Subject: [PATCH 34/86] Bump core version

---
 cmds/portmaster-core/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index 2b113ab3..0421a1b3 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.10", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.11", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")

From f5d89c2aed14cab27433f13a4d8ca257d1e99dd4 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 19 May 2022 11:30:34 +0200
Subject: [PATCH 35/86] Update SPN

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 6fa2cb23..43e69e3c 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.49
 	github.com/oschwald/maxminddb-golang v1.9.0
 	github.com/safing/portbase v0.14.4
-	github.com/safing/spn v0.4.9
+	github.com/safing/spn v0.4.10
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
diff --git a/go.sum b/go.sum
index fa786ada..4b2d7c17 100644
--- a/go.sum
+++ b/go.sum
@@ -858,6 +858,8 @@ github.com/safing/spn v0.4.8 h1:C5QOl03hypDaC4qvFtWjAKmSJIOjfqYVGT+sTnUIO/E=
 github.com/safing/spn v0.4.8/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.9 h1:p+TRS2y1yAY4CYyHiVuy+V3NcA9p826NtYksuBaSl64=
 github.com/safing/spn v0.4.9/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
+github.com/safing/spn v0.4.10 h1:nZuWPBn1z0aBUDnnpG9pKZCmPAH2hgNE4cqiSB7JGU8=
+github.com/safing/spn v0.4.10/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

From bc745d4399a750da6956fc4e350a5f5dfe2a5f3f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 19 May 2022 11:30:45 +0200
Subject: [PATCH 36/86] Bump core version

---
 cmds/portmaster-core/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index 0421a1b3..3b4e7b04 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.11", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.12", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")

From ea1b189330d1ed4749f4044bc646681beb91720f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 20 May 2022 16:35:46 +0200
Subject: [PATCH 37/86] Reset system self-check after network change

---
 compat/module.go | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/compat/module.go b/compat/module.go
index d1a27628..77478f7e 100644
--- a/compat/module.go
+++ b/compat/module.go
@@ -26,6 +26,10 @@ var (
 	// selfcheckFails counts how often the self check failed successively.
 	// selfcheckFails is not locked as it is only accessed by the self-check task.
 	selfcheckFails int
+
+	// selfcheckNetworkChangedFlag is used to track changed to the network for
+	// the self-check.
+	selfcheckNetworkChangedFlag = netenv.GetNetworkChangedFlag()
 )
 
 // selfcheckFailThreshold holds the threshold of how many times the selfcheck
@@ -49,6 +53,7 @@ func prep() error {
 func start() error {
 	startNotify()
 
+	selfcheckNetworkChangedFlag.Refresh()
 	selfcheckTask = module.NewTask("compatibility self-check", selfcheckTaskFunc).
 		Repeat(5 * time.Minute).
 		MaxDelay(selfcheckTaskRetryAfter).
@@ -83,19 +88,23 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 
 	// Run selfcheck and return if successful.
 	issue, err := selfcheck(ctx)
-	if err == nil {
-		selfCheckIsFailing.UnSet()
-		selfcheckFails = 0
-		resetSystemIssue()
+	switch {
+	case err == nil:
+		// Successful.
 		tracer.Debugf("compat: self-check successful")
-		return nil
-	}
+	case issue == nil:
+		// Internal error.
+		tracer.Warningf("compat: %s", err)
+	case selfcheckNetworkChangedFlag.IsSet():
+		// The network changed, ignore the issue.
+	default:
+		// The self-check failed.
 
-	// Log result.
-	if issue != nil {
+		// Set state and increase counter.
 		selfCheckIsFailing.Set()
 		selfcheckFails++
 
+		// Log and notify.
 		tracer.Errorf("compat: %s", err)
 		if selfcheckFails >= selfcheckFailThreshold {
 			issue.notify(err)
@@ -103,14 +112,16 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 
 		// Retry quicker when failed.
 		task.Schedule(time.Now().Add(selfcheckTaskRetryAfter))
-	} else {
-		selfCheckIsFailing.UnSet()
-		selfcheckFails = 0
 
-		// Only log internal errors, but don't notify.
-		tracer.Warningf("compat: %s", err)
+		return nil
 	}
 
+	// Reset self-check state.
+	selfcheckNetworkChangedFlag.Refresh()
+	selfCheckIsFailing.UnSet()
+	selfcheckFails = 0
+	resetSystemIssue()
+
 	return nil
 }
 

From e178b732bca2a3daa0f5ab67e8e01a0500ccd4e6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 20 May 2022 16:37:19 +0200
Subject: [PATCH 38/86] Calm down and relax dns query check

---
 firewall/tunnel.go      |  2 +-
 netenv/online-status.go | 76 +++++++++++++++++++----------------------
 resolver/resolve.go     | 48 ++++++++++++++++++++++++--
 resolver/resolver.go    | 12 ++++++-
 4 files changed, 94 insertions(+), 44 deletions(-)

diff --git a/firewall/tunnel.go b/firewall/tunnel.go
index 32d60de1..08adff33 100644
--- a/firewall/tunnel.go
+++ b/firewall/tunnel.go
@@ -34,7 +34,7 @@ func checkTunneling(ctx context.Context, conn *network.Connection, pkt packet.Pa
 	case conn.Process().Pid == ownPID:
 		// Bypass tunneling for certain own connections.
 		switch {
-		case captain.ClientBootstrapping():
+		case !captain.ClientReady():
 			return
 		case captain.IsExcepted(conn.Entity.IP):
 			return
diff --git a/netenv/online-status.go b/netenv/online-status.go
index a7589a13..0ef612fd 100644
--- a/netenv/online-status.go
+++ b/netenv/online-status.go
@@ -37,13 +37,12 @@ var (
 	PortalTestIP  = net.IPv4(192, 0, 2, 1)
 	PortalTestURL = fmt.Sprintf("http://%s/", PortalTestIP)
 
-	DNSTestDomain     = "one.one.one.one."
-	DNSTestExpectedIP = net.IPv4(1, 1, 1, 1)
-
-	DNSFallbackTestDomain     = "dns-check.safing.io."
-	DNSFallbackTestExpectedIP = net.IPv4(0, 65, 67, 75) // Ascii: \0ACK
+	DNSTestDomain     = "online-check.safing.io."
+	DNSTestExpectedIP = net.IPv4(0, 65, 67, 75) // Ascii: \0ACK
+	DNSTestQueryFunc  func(ctx context.Context, fdqn string) (ips []net.IP, ok bool, err error)
 
 	ConnectedToSPN = abool.New()
+	ConnectedToDNS = abool.New()
 
 	// SpecialCaptivePortalDomain is the domain name used to point to the detected captive portal IP
 	// or the captive portal test IP. The default value should be overridden by the resolver package,
@@ -53,8 +52,6 @@ var (
 	// ConnectivityDomains holds all connectivity domains. This slice must not be modified.
 	ConnectivityDomains = []string{
 		SpecialCaptivePortalDomain,
-		DNSTestDomain,         // Internal DNS Check
-		DNSFallbackTestDomain, // Internal DNS Check
 
 		// Windows
 		"dns.msftncsi.com.", // DNS Check
@@ -380,20 +377,20 @@ func monitorOnlineStatus(ctx context.Context) error {
 func getDynamicStatusTrigger() <-chan time.Time {
 	switch GetOnlineStatus() {
 	case StatusOffline:
-		// Will be triggered by network change anyway.
-		return time.After(20 * time.Second)
+		// Will also be triggered by network change.
+		return time.After(10 * time.Second)
 	case StatusLimited, StatusPortal:
 		// Change will not be detected otherwise, but impact is minor.
 		return time.After(5 * time.Second)
 	case StatusSemiOnline:
 		// Very small impact.
-		return time.After(20 * time.Second)
+		return time.After(60 * time.Second)
 	case StatusOnline:
 		// Don't check until resolver reports problems.
 		return nil
 	case StatusUnknown:
-		return time.After(5 * time.Second)
-	default: // other unknown status
+		fallthrough
+	default:
 		return time.After(5 * time.Minute)
 	}
 }
@@ -407,13 +404,18 @@ func checkOnlineStatus(ctx context.Context) {
 		return StatusUnknown
 	}*/
 
-	// 0) check if connected to SPN
+	// 0) check if connected to SPN and/or DNS.
 
 	if ConnectedToSPN.IsSet() {
 		updateOnlineStatus(StatusOnline, nil, "connected to SPN")
 		return
 	}
 
+	if ConnectedToDNS.IsSet() {
+		updateOnlineStatus(StatusOnline, nil, "connected to DNS")
+		return
+	}
+
 	// 1) check for addresses
 
 	ipv4, ipv6, err := GetAssignedAddresses()
@@ -508,34 +510,28 @@ func checkOnlineStatus(ctx context.Context) {
 
 	// 3) resolve a query
 
-	// Check with primary dns check domain.
-	ips, err := net.LookupIP(DNSTestDomain)
-	if err != nil {
-		log.Warningf("netenv: dns check query failed: %s", err)
-	} else {
-		// check for expected response
-		for _, ip := range ips {
-			if ip.Equal(DNSTestExpectedIP) {
-				updateOnlineStatus(StatusOnline, nil, "all checks passed")
-				return
-			}
-		}
-	}
-
-	// If that did not work, check with fallback dns check domain.
-	ips, err = net.LookupIP(DNSFallbackTestDomain)
-	if err != nil {
-		log.Warningf("netenv: dns fallback check query failed: %s", err)
-		updateOnlineStatus(StatusLimited, nil, "dns fallback check query failed")
+	// Check if we can resolve the dns check domain.
+	if DNSTestQueryFunc == nil {
+		updateOnlineStatus(StatusOnline, nil, "all checks passed, dns query check disabled")
 		return
 	}
-	// check for expected response
-	for _, ip := range ips {
-		if ip.Equal(DNSFallbackTestExpectedIP) {
-			updateOnlineStatus(StatusOnline, nil, "all checks passed")
-			return
-		}
+	ips, ok, err := DNSTestQueryFunc(ctx, DNSTestDomain)
+	switch {
+	case ok && err != nil:
+		updateOnlineStatus(StatusOnline, nil, fmt.Sprintf(
+			"all checks passed, acceptable result for dns query check: %s",
+			err,
+		))
+	case ok && len(ips) >= 1 && ips[0].Equal(DNSTestExpectedIP):
+		updateOnlineStatus(StatusOnline, nil, "all checks passed")
+	case ok && len(ips) >= 1:
+		log.Warningf("netenv: dns query check response mismatched: got %s", ips[0])
+		updateOnlineStatus(StatusOnline, nil, "all checks passed, dns query check response mismatched")
+	case ok:
+		log.Warningf("netenv: dns query check response mismatched: empty response")
+		updateOnlineStatus(StatusOnline, nil, "all checks passed, dns query check response was empty")
+	default:
+		log.Warningf("netenv: dns query check failed: %s", err)
+		updateOnlineStatus(StatusOffline, nil, "dns query check failed")
 	}
-	// unexpected response
-	updateOnlineStatus(StatusSemiOnline, nil, "dns check query response mismatched")
 }
diff --git a/resolver/resolve.go b/resolver/resolve.go
index 8663a32e..1b0b429e 100644
--- a/resolver/resolve.go
+++ b/resolver/resolve.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"sync"
 	"time"
 
@@ -318,8 +319,8 @@ func resolveAndCache(ctx context.Context, q *Query, oldCache *RRCache) (rrCache
 	}
 
 	// check if we are online
-	if primarySource != ServerSourceEnv && netenv.GetOnlineStatus() == netenv.StatusOffline {
-		if !netenv.IsConnectivityDomain(q.FQDN) {
+	if netenv.GetOnlineStatus() == netenv.StatusOffline && primarySource != ServerSourceEnv {
+		if q.FQDN != netenv.DNSTestDomain && !netenv.IsConnectivityDomain(q.FQDN) {
 			// we are offline and this is not an online check query
 			return oldCache, ErrOffline
 		}
@@ -358,6 +359,7 @@ resolveLoop:
 					// some resolvers might also block
 					return nil, err
 				case netenv.GetOnlineStatus() == netenv.StatusOffline &&
+					q.FQDN != netenv.DNSTestDomain &&
 					!netenv.IsConnectivityDomain(q.FQDN):
 					// we are offline and this is not an online check query
 					return oldCache, ErrOffline
@@ -478,3 +480,45 @@ func shouldResetCache(q *Query) (reset bool) {
 
 	return false
 }
+
+func init() {
+	netenv.DNSTestQueryFunc = testConnectivity
+}
+
+// testConnectivity test if resolving a query succeeds and returns whether the
+// query itself succeeded, separate from interpreting the result.
+func testConnectivity(ctx context.Context, fdqn string) (ips []net.IP, ok bool, err error) {
+	q := &Query{
+		FQDN:      fdqn,
+		QType:     dns.Type(dns.TypeA),
+		NoCaching: true,
+	}
+	if !q.check() {
+		return nil, false, ErrInvalid
+	}
+
+	rrCache, err := resolveAndCache(ctx, q, nil)
+	switch {
+	case err == nil:
+		switch rrCache.RCode {
+		case dns.RcodeNameError:
+			return nil, true, ErrNotFound
+		case dns.RcodeRefused:
+			return nil, true, errors.New("refused")
+		default:
+			ips := rrCache.ExportAllARecords()
+			if len(ips) > 0 {
+				return ips, true, nil
+			}
+			return nil, true, ErrNotFound
+		}
+	case errors.Is(err, ErrNotFound):
+		return nil, true, err
+	case errors.Is(err, ErrBlocked):
+		return nil, true, err
+	case errors.Is(err, ErrNoCompliance):
+		return nil, true, err
+	default:
+		return nil, false, err
+	}
+}
diff --git a/resolver/resolver.go b/resolver/resolver.go
index 56d4b6cc..1f12c8f2 100644
--- a/resolver/resolver.go
+++ b/resolver/resolver.go
@@ -206,8 +206,8 @@ func (brc *BasicResolverConn) init() {
 
 // ReportFailure reports that an error occurred with this resolver.
 func (brc *BasicResolverConn) ReportFailure() {
+	// Don't mark resolver as failed if we are offline.
 	if !netenv.Online() {
-		// don't mark failed if we are offline
 		return
 	}
 
@@ -224,6 +224,11 @@ func (brc *BasicResolverConn) ReportFailure() {
 		// the fail.
 		brc.networkChangedFlag.Refresh()
 	}
+
+	// Report to netenv that a configured server failed.
+	if brc.resolver.Info.Source == ServerSourceConfigured {
+		netenv.ConnectedToDNS.UnSet()
+	}
 }
 
 // IsFailing returns if this resolver is currently failing.
@@ -255,4 +260,9 @@ func (brc *BasicResolverConn) ResetFailure() {
 		defer brc.failLock.Unlock()
 		brc.fails = 0
 	}
+
+	// Report to netenv that a configured server succeeded.
+	if brc.resolver.Info.Source == ServerSourceConfigured {
+		netenv.ConnectedToDNS.Set()
+	}
 }

From 49e79fe3fdd3b5f3a03a3a36e398d8604b9922bc Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 24 May 2022 11:18:23 +0200
Subject: [PATCH 39/86] Detect responses to multi/broadcast queries

---
 firewall/master.go     | 43 ++++++++++++++++++++++++++++++++++++++++++
 firewall/tunnel.go     |  4 ++--
 netenv/addresses.go    | 14 ++++++--------
 network/multicast.go   | 43 ++++++++++++++++++++++++++++++++++++++++++
 network/netutils/ip.go | 31 ++++++++++++++++++++++++++++++
 process/process.go     | 26 +++++++++++++++++++++++++
 6 files changed, 151 insertions(+), 10 deletions(-)
 create mode 100644 network/multicast.go

diff --git a/firewall/master.go b/firewall/master.go
index 16dc2b5f..4be457aa 100644
--- a/firewall/master.go
+++ b/firewall/master.go
@@ -3,7 +3,9 @@ package firewall
 import (
 	"context"
 	"fmt"
+	"net"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/agext/levenshtein"
@@ -41,6 +43,7 @@ type deciderFn func(context.Context, *network.Connection, *profile.LayeredProfil
 var defaultDeciders = []deciderFn{
 	checkPortmasterConnection,
 	checkSelfCommunication,
+	checkIfBroadcastReply,
 	checkConnectionType,
 	checkConnectionScope,
 	checkEndpointLists,
@@ -182,6 +185,46 @@ func checkSelfCommunication(ctx context.Context, conn *network.Connection, _ *pr
 	return false
 }
 
+func checkIfBroadcastReply(ctx context.Context, conn *network.Connection, _ *profile.LayeredProfile, _ packet.Packet) bool {
+	// Only check inbound connections.
+	if !conn.Inbound {
+		return false
+	}
+	// Only check if the process has been identified.
+	if !conn.Process().IsIdentified() {
+		return false
+	}
+
+	// Check if the remote IP is part of a local network.
+	localNet, err := netenv.GetLocalNetwork(conn.Entity.IP)
+	if err != nil {
+		log.Tracer(ctx).Warningf("filter: failed to get local network: %s", err)
+		return false
+	}
+	if localNet == nil {
+		return false
+	}
+
+	// Search for a matching requesting connection.
+	requestingConn := network.GetMulticastRequestConn(conn, localNet)
+	if requestingConn == nil {
+		return false
+	}
+
+	conn.Accept(
+		fmt.Sprintf(
+			"response to multi/broadcast query to %s/%s",
+			packet.IPProtocol(requestingConn.Entity.Protocol),
+			net.JoinHostPort(
+				requestingConn.Entity.IP.String(),
+				strconv.Itoa(int(requestingConn.Entity.Port)),
+			),
+		),
+		"",
+	)
+	return true
+}
+
 func checkEndpointLists(ctx context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
 	// DNS request from the system resolver require a special decision process,
 	// because the original requesting process is not known. Here, we only check
diff --git a/firewall/tunnel.go b/firewall/tunnel.go
index 08adff33..28f0e37d 100644
--- a/firewall/tunnel.go
+++ b/firewall/tunnel.go
@@ -42,10 +42,10 @@ func checkTunneling(ctx context.Context, conn *network.Connection, pkt packet.Pa
 	}
 
 	// Check more extensively for Local/LAN connections.
-	myNet, err := netenv.IsMyNet(conn.Entity.IP)
+	localNet, err := netenv.GetLocalNetwork(conn.Entity.IP)
 	if err != nil {
 		log.Warningf("firewall: failed to check if %s is in my net: %s", conn.Entity.IP, err)
-	} else if myNet {
+	} else if localNet != nil {
 		// With IPv6, just checking the IP scope is not enough, as the host very
 		// likely has a public IPv6 address.
 		// Don't tunnel LAN connections.
diff --git a/netenv/addresses.go b/netenv/addresses.go
index 14762c78..67c4c999 100644
--- a/netenv/addresses.go
+++ b/netenv/addresses.go
@@ -152,11 +152,9 @@ func IsMyIP(ip net.IP) (yes bool, err error) {
 	return false, nil
 }
 
-// IsMyNet returns whether the given IP is currently in the host's broadcast
-// domain - ie. the networks that the host is directly attached to.
-// Function is optimized with the assumption that is unlikely that the IP is
-// in the broadcast domain.
-func IsMyNet(ip net.IP) (yes bool, err error) {
+// GetLocalNetwork uses the given IP to search for a network configured on the
+// device and returns it.
+func GetLocalNetwork(ip net.IP) (myNet *net.IPNet, err error) {
 	myNetworksLock.Lock()
 	defer myNetworksLock.Unlock()
 
@@ -164,16 +162,16 @@ func IsMyNet(ip net.IP) (yes bool, err error) {
 	if myNetworksNetworkChangedFlag.IsSet() {
 		err := refreshMyNetworks()
 		if err != nil {
-			return false, err
+			return nil, err
 		}
 	}
 
 	// Check if the IP address is in my networks.
 	for _, myNet := range myNetworks {
 		if myNet.Contains(ip) {
-			return true, nil
+			return myNet, nil
 		}
 	}
 
-	return false, nil
+	return nil, nil
 }
diff --git a/network/multicast.go b/network/multicast.go
new file mode 100644
index 00000000..82cb2173
--- /dev/null
+++ b/network/multicast.go
@@ -0,0 +1,43 @@
+package network
+
+import (
+	"net"
+
+	"github.com/safing/portmaster/network/netutils"
+)
+
+// GetMulticastRequestConn searches for and returns the requesting connnection
+// of a possible multicast/broadcast response.
+func GetMulticastRequestConn(responseConn *Connection, responseFromNet *net.IPNet) *Connection {
+	// Calculate the broadcast address the query would have gone to.
+	responseNetBroadcastIP := netutils.GetBroadcastAddress(responseFromNet.IP, responseFromNet.Mask)
+
+	// Find requesting multicast/broadcast connection.
+	for _, conn := range conns.clone() {
+		switch {
+		case conn.Inbound:
+			// Ignore incoming connections.
+		//case conn.Ended != 0:
+		// Ignore ended connections.
+		case conn.Entity.Protocol != responseConn.Entity.Protocol:
+			// Ignore on protocol mismatch.
+		case conn.LocalPort != responseConn.LocalPort:
+			// Ignore on local port mismatch.
+		case !conn.LocalIP.Equal(responseConn.LocalIP):
+			// Ignore on local IP mismatch.
+		case !conn.Process().Equal(responseConn.Process()):
+			// Ignore if processes mismatch.
+		case conn.Entity.IPScope == netutils.LocalMulticast &&
+			(responseConn.Entity.IPScope == netutils.LinkLocal ||
+				responseConn.Entity.IPScope == netutils.SiteLocal):
+			// We found a (possibly routed) multicast request that matches the response!
+			return conn
+		case conn.Entity.IP.Equal(responseNetBroadcastIP) &&
+			responseFromNet.Contains(conn.LocalIP):
+			// We found a (link local) broadcast request that matches the response!
+			return conn
+		}
+	}
+
+	return nil
+}
diff --git a/network/netutils/ip.go b/network/netutils/ip.go
index f6a34003..76bb0230 100644
--- a/network/netutils/ip.go
+++ b/network/netutils/ip.go
@@ -28,6 +28,9 @@ func GetIPScope(ip net.IP) IPScope { //nolint:gocognit
 	if ip4 := ip.To4(); ip4 != nil {
 		// IPv4
 		switch {
+		case ip4[0] == 0:
+			// 0.0.0.0/8
+			return Invalid
 		case ip4[0] == 127:
 			// 127.0.0.0/8 (RFC1918)
 			return HostLocal
@@ -79,6 +82,8 @@ func GetIPScope(ip net.IP) IPScope { //nolint:gocognit
 	} else if len(ip) == net.IPv6len {
 		// IPv6
 		switch {
+		case ip.Equal(net.IPv6zero):
+			return Invalid
 		case ip.Equal(net.IPv6loopback):
 			return HostLocal
 		case ip[0]&0xfe == 0xfc:
@@ -124,3 +129,29 @@ func (scope IPScope) IsGlobal() bool {
 		return false
 	}
 }
+
+// GetBroadcastAddress returns the broadcast address of the given IP and network mask.
+// If a mixed IPv4/IPv6 input is given, it returns nil.
+func GetBroadcastAddress(ip net.IP, netMask net.IPMask) net.IP {
+	// Convert to standard v4.
+	if ip4 := ip.To4(); ip4 != nil {
+		ip = ip4
+	}
+	mask := net.IP(netMask)
+	if ip4Mask := mask.To4(); ip4Mask != nil {
+		mask = ip4Mask
+	}
+
+	// Check for mixed v4/v6 input.
+	if len(ip) != len(mask) {
+		return nil
+	}
+
+	// Merge to broadcast address
+	n := len(ip)
+	broadcastAddress := make(net.IP, n)
+	for i := 0; i < n; i++ {
+		broadcastAddress[i] = ip[i] | ^mask[i]
+	}
+	return broadcastAddress
+}
diff --git a/process/process.go b/process/process.go
index 8bae2772..ff04896e 100644
--- a/process/process.go
+++ b/process/process.go
@@ -67,6 +67,32 @@ func (p *Process) Profile() *profile.LayeredProfile {
 	return p.profile
 }
 
+// IsIdentified returns whether the process has been identified or if it
+// represents some kind of unidentified process.
+func (p *Process) IsIdentified() bool {
+	// Check if process exists.
+	if p == nil {
+		return false
+	}
+
+	// Check for special PIDs.
+	switch p.Pid {
+	case UndefinedProcessID:
+		return false
+	case UnidentifiedProcessID:
+		return false
+	case UnsolicitedProcessID:
+		return false
+	default:
+		return true
+	}
+}
+
+// Equal returns if the two processes are both identified and have the same PID.
+func (p *Process) Equal(other *Process) bool {
+	return p.IsIdentified() && other.IsIdentified() && p.Pid == other.Pid
+}
+
 // IsSystemResolver is a shortcut to check if the process is or belongs to the
 // system resolver and needs special handling.
 func (p *Process) IsSystemResolver() bool {

From 9a89f65027d923a0e44ff6ec2fcff0df514b9f75 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 24 May 2022 11:21:11 +0200
Subject: [PATCH 40/86] Improve support for DNS-SD and fall back to cached data
 for non-ICANN queries

---
 firewall/master.go        |  5 +--
 nameserver/nameserver.go  | 73 +++++++++++++++++++++++++++++++++++----
 network/netutils/dns.go   | 52 +++++++++++++++++++---------
 resolver/resolve.go       | 42 ++++++++++++++++++++++
 resolver/resolver-env.go  | 11 +++++-
 resolver/resolver_test.go | 56 ++++++++++++++++++++++++++++++
 6 files changed, 211 insertions(+), 28 deletions(-)

diff --git a/firewall/master.go b/firewall/master.go
index 4be457aa..e50617d0 100644
--- a/firewall/master.go
+++ b/firewall/master.go
@@ -482,10 +482,7 @@ func checkDomainHeuristics(ctx context.Context, conn *network.Connection, p *pro
 	trimmedDomain := strings.TrimRight(conn.Entity.Domain, ".")
 	etld1, err := publicsuffix.EffectiveTLDPlusOne(trimmedDomain)
 	if err != nil {
-		// we don't apply any checks here and let the request through
-		// because a malformed domain-name will likely be dropped by
-		// checks better suited for that.
-		log.Tracer(ctx).Warningf("filter: failed to get eTLD+1: %s", err)
+		// Don't run the check if the domain is a TLD.
 		return false
 	}
 
diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go
index a92ca8ad..afdae60b 100644
--- a/nameserver/nameserver.go
+++ b/nameserver/nameserver.go
@@ -106,6 +106,9 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 		return reply(nsutil.Refused("invalid domain"))
 	}
 
+	// Get public suffix after validation.
+	q.InitPublicSuffixData()
+
 	// Check if query is failing.
 	// Some software retries failing queries excessively. This might not be a
 	// problem normally, but handling a request is pretty expensive for the
@@ -252,10 +255,13 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 	if err != nil {
 		switch {
 		case errors.Is(err, resolver.ErrNotFound):
-			tracer.Tracef("nameserver: %s", err)
-			conn.Failed("domain does not exist", "")
-			return reply(nsutil.NxDomain("nxdomain: " + err.Error()))
-
+			// Try alternatives domain names for unofficial domain spaces.
+			rrCache = checkAlternativeCaches(ctx, q)
+			if rrCache == nil {
+				tracer.Tracef("nameserver: %s", err)
+				conn.Failed("domain does not exist", "")
+				return reply(nsutil.NxDomain("nxdomain: " + err.Error()))
+			}
 		case errors.Is(err, resolver.ErrBlocked):
 			tracer.Tracef("nameserver: %s", err)
 			conn.Block(err.Error(), "")
@@ -268,7 +274,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 
 		case errors.Is(err, resolver.ErrOffline):
 			if rrCache == nil {
-				log.Tracer(ctx).Debugf("nameserver: not resolving %s, device is offline", q.ID())
+				tracer.Debugf("nameserver: not resolving %s, device is offline", q.ID())
 				conn.Failed("not resolving, device is offline", "")
 				return reply(nsutil.ServerFailure(err.Error()))
 			}
@@ -290,8 +296,12 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 		addFailingQuery(q, errors.New("emptry reply from resolver"))
 		return reply(nsutil.ServerFailure("internal error: empty reply"))
 	case rrCache.RCode == dns.RcodeNameError:
-		// Return now if NXDomain.
-		return reply(nsutil.NxDomain("no answer found (NXDomain)"))
+		// Try alternatives domain names for unofficial domain spaces.
+		rrCache = checkAlternativeCaches(ctx, q)
+		if rrCache == nil {
+			// Return now if NXDomain.
+			return reply(nsutil.NxDomain("no answer found (NXDomain)"))
+		}
 	}
 
 	// Check with firewall again after resolving.
@@ -336,3 +346,52 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, request *dns.Msg)
 	)
 	return reply(rrCache, conn, rrCache)
 }
+
+func checkAlternativeCaches(ctx context.Context, q *resolver.Query) *resolver.RRCache {
+	// Do not try alternatives when the query is in a public suffix.
+	// This also includes arpa. and local.
+	if q.ICANNSpace {
+		return nil
+	}
+
+	// Check if the env resolver has something.
+	pmEnvQ := &resolver.Query{
+		FQDN:  q.FQDN + "local." + resolver.InternalSpecialUseDomain,
+		QType: q.QType,
+	}
+	rrCache, err := resolver.QueryPortmasterEnv(ctx, pmEnvQ)
+	if err == nil && rrCache != nil && rrCache.RCode == dns.RcodeSuccess {
+		makeAlternativeRecord(ctx, q, rrCache, pmEnvQ.FQDN)
+		return rrCache
+	}
+
+	// Check if we have anything in cache
+	localFQDN := q.FQDN + "local."
+	rrCache, err = resolver.GetRRCache(localFQDN, q.QType)
+	if err == nil && rrCache != nil && rrCache.RCode == dns.RcodeSuccess {
+		makeAlternativeRecord(ctx, q, rrCache, localFQDN)
+		return rrCache
+	}
+
+	return nil
+}
+
+func makeAlternativeRecord(ctx context.Context, q *resolver.Query, rrCache *resolver.RRCache, altName string) {
+	log.Tracer(ctx).Debugf("using %s to answer query", altName)
+
+	// Duplicate answers so they match the query.
+	copied := make([]dns.RR, 0, len(rrCache.Answer))
+	for _, answer := range rrCache.Answer {
+		if strings.ToLower(answer.Header().Name) == altName {
+			copiedAnswer := dns.Copy(answer)
+			copiedAnswer.Header().Name = q.FQDN
+			copied = append(copied, copiedAnswer)
+		}
+	}
+	if len(copied) > 0 {
+		rrCache.Answer = append(rrCache.Answer, copied...)
+	}
+
+	// Update the question.
+	rrCache.Domain = q.FQDN
+}
diff --git a/network/netutils/dns.go b/network/netutils/dns.go
index 7a7bd6e5..252dca47 100644
--- a/network/netutils/dns.go
+++ b/network/netutils/dns.go
@@ -4,21 +4,38 @@ import (
 	"fmt"
 	"net"
 	"regexp"
+	"strings"
 
 	"github.com/miekg/dns"
 )
 
-var cleanDomainRegex = regexp.MustCompile(
-	`^` + // match beginning
-		`(` + // start subdomain group
-		`(xn--)?` + // idn prefix
-		`[a-z0-9_-]{1,63}` + // main chunk
-		`\.` + // ending with a dot
-		`)*` + // end subdomain group, allow any number of subdomains
-		`(xn--)?` + // TLD idn prefix
-		`[a-z0-9_-]{2,63}` + // TLD main chunk with at least two characters
-		`\.` + // ending with a dot
-		`$`, // match end
+var (
+	cleanDomainRegex = regexp.MustCompile(
+		`^` + // match beginning
+			`(` + // start subdomain group
+			`(xn--)?` + // idn prefix
+			`[a-z0-9_-]{1,63}` + // main chunk
+			`\.` + // ending with a dot
+			`)*` + // end subdomain group, allow any number of subdomains
+			`(xn--)?` + // TLD idn prefix
+			`[a-z0-9_-]{2,63}` + // TLD main chunk with at least two characters
+			`\.` + // ending with a dot
+			`$`, // match end
+	)
+
+	// dnsSDDomainRegex is a lot more lax to better suit the allowed characters in DNS-SD.
+	// Not all characters have been allowed - some special characters were
+	// removed to reduce the general attack surface.
+	dnsSDDomainRegex = regexp.MustCompile(
+		// Start of charset selection.
+		`^[` +
+			// Printable ASCII (character code 32-127), excluding some special characters.
+			` !#$%&()*+,\-\./0-9:;=?@A-Z[\\\]^_\a-z{|}~` +
+			// Only latin characters from extended ASCII (character code 128-255).
+			`ŠŒŽšœžŸ¡¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ` +
+			// End of charset selection.
+			`]*$`,
+	)
 )
 
 // IsValidFqdn returns whether the given string is a valid fqdn.
@@ -33,15 +50,18 @@ func IsValidFqdn(fqdn string) bool {
 		return false
 	}
 
-	// check with regex
-	if !cleanDomainRegex.MatchString(fqdn) {
+	// IsFqdn checks if a domain name is fully qualified.
+	if !dns.IsFqdn(fqdn) {
 		return false
 	}
 
-	// check with miegk/dns
+	// Use special check for .local domains to support DNS-SD.
+	if strings.HasSuffix(fqdn, ".local.") {
+		return dnsSDDomainRegex.MatchString(fqdn)
+	}
 
-	// IsFqdn checks if a domain name is fully qualified.
-	if !dns.IsFqdn(fqdn) {
+	// check with regex
+	if !cleanDomainRegex.MatchString(fqdn) {
 		return false
 	}
 
diff --git a/resolver/resolve.go b/resolver/resolve.go
index 1b0b429e..9f7fad66 100644
--- a/resolver/resolve.go
+++ b/resolver/resolve.go
@@ -5,10 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/miekg/dns"
+	"golang.org/x/net/publicsuffix"
 
 	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/log"
@@ -90,6 +92,11 @@ type Query struct {
 	IgnoreFailing      bool
 	LocalResolversOnly bool
 
+	// ICANNSpace signifies if the domain is within ICANN managed domain space.
+	ICANNSpace bool
+	// Domain root is the effective TLD +1.
+	DomainRoot string
+
 	// internal
 	dotPrefixedFQDN string
 }
@@ -99,6 +106,41 @@ func (q *Query) ID() string {
 	return q.FQDN + q.QType.String()
 }
 
+// InitPublicSuffixData initializes the public suffix data.
+func (q *Query) InitPublicSuffixData() {
+	// Get public suffix and derive if domain is in ICANN space.
+	suffix, icann := publicsuffix.PublicSuffix(strings.TrimSuffix(q.FQDN, "."))
+	if icann || strings.Contains(suffix, ".") {
+		q.ICANNSpace = true
+	}
+	// Override some cases.
+	switch suffix {
+	case "example":
+		q.ICANNSpace = true // Defined by ICANN.
+	case "invalid":
+		q.ICANNSpace = true // Defined by ICANN.
+	case "local":
+		q.ICANNSpace = true // Defined by ICANN.
+	case "localhost":
+		q.ICANNSpace = true // Defined by ICANN.
+	case "onion":
+		q.ICANNSpace = false // Defined by ICANN, but special.
+	case "test":
+		q.ICANNSpace = true // Defined by ICANN.
+	}
+	// Add suffix to adhere to FQDN format.
+	suffix += "."
+
+	switch {
+	case len(q.FQDN) == len(suffix):
+		// We are at or below the domain root, reset.
+		q.DomainRoot = ""
+	case len(q.FQDN) > len(suffix):
+		domainRootStart := strings.LastIndex(q.FQDN[:len(q.FQDN)-len(suffix)-1], ".") + 1
+		q.DomainRoot = q.FQDN[domainRootStart:]
+	}
+}
+
 // check runs sanity checks and does some initialization. Returns whether the query passed the basic checks.
 func (q *Query) check() (ok bool) {
 	if q.FQDN == "" {
diff --git a/resolver/resolver-env.go b/resolver/resolver-env.go
index 18acfba4..b8710ff3 100644
--- a/resolver/resolver-env.go
+++ b/resolver/resolver-env.go
@@ -128,7 +128,7 @@ func (er *envResolverConn) makeRRCache(q *Query, answers []dns.RR) *RRCache {
 	// Disable caching, as the env always has the raw data available.
 	q.NoCaching = true
 
-	return &RRCache{
+	rrCache := &RRCache{
 		Domain:   q.FQDN,
 		Question: q.QType,
 		RCode:    dns.RcodeSuccess,
@@ -136,6 +136,10 @@ func (er *envResolverConn) makeRRCache(q *Query, answers []dns.RR) *RRCache {
 		Extra:    []dns.RR{internalSpecialUseComment}, // Always add comment about this TLD.
 		Resolver: envResolver.Info.Copy(),
 	}
+	if len(rrCache.Answer) == 0 {
+		rrCache.RCode = dns.RcodeNameError
+	}
+	return rrCache
 }
 
 func (er *envResolverConn) ReportFailure() {}
@@ -145,3 +149,8 @@ func (er *envResolverConn) IsFailing() bool {
 }
 
 func (er *envResolverConn) ResetFailure() {}
+
+// QueryPortmasterEnv queries the environment resolver directly.
+func QueryPortmasterEnv(ctx context.Context, q *Query) (*RRCache, error) {
+	return envResolver.Conn.Query(ctx, q)
+}
diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go
index f385e52e..61e05376 100644
--- a/resolver/resolver_test.go
+++ b/resolver/resolver_test.go
@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/miekg/dns"
 
 	"github.com/safing/portbase/log"
@@ -103,3 +105,57 @@ func TestBulkResolving(t *testing.T) {
 
 	t.Logf("total time taken: %s", time.Since(started))
 }
+
+func TestPublicSuffix(t *testing.T) {
+	t.Parallel()
+
+	testSuffix(t, "co.uk.", "", true)
+	testSuffix(t, "amazon.co.uk.", "amazon.co.uk.", true)
+	testSuffix(t, "books.amazon.co.uk.", "amazon.co.uk.", true)
+	testSuffix(t, "www.books.amazon.co.uk.", "amazon.co.uk.", true)
+	testSuffix(t, "com.", "", true)
+	testSuffix(t, "amazon.com.", "amazon.com.", true)
+	testSuffix(t, "example0.debian.net.", "example0.debian.net.", true)
+	testSuffix(t, "example1.debian.org.", "debian.org.", true)
+	testSuffix(t, "golang.dev.", "golang.dev.", true)
+	testSuffix(t, "golang.net.", "golang.net.", true)
+	testSuffix(t, "play.golang.org.", "golang.org.", true)
+	testSuffix(t, "gophers.in.space.museum.", "in.space.museum.", true)
+	testSuffix(t, "0emm.com.", "0emm.com.", true)
+	testSuffix(t, "a.0emm.com.", "", true)
+	testSuffix(t, "b.c.d.0emm.com.", "c.d.0emm.com.", true)
+	testSuffix(t, "org.", "", true)
+	testSuffix(t, "foo.org.", "foo.org.", true)
+	testSuffix(t, "foo.co.uk.", "foo.co.uk.", true)
+	testSuffix(t, "foo.dyndns.org.", "foo.dyndns.org.", true)
+	testSuffix(t, "foo.blogspot.co.uk.", "foo.blogspot.co.uk.", true)
+	testSuffix(t, "there.is.no.such-tld.", "no.such-tld.", false)
+	testSuffix(t, "www.some.bit.", "some.bit.", false)
+	testSuffix(t, "cromulent.", "", false)
+	testSuffix(t, "arpa.", "", true)
+	testSuffix(t, "in-addr.arpa.", "", true)
+	testSuffix(t, "1.in-addr.arpa.", "1.in-addr.arpa.", true)
+	testSuffix(t, "ip6.arpa.", "", true)
+	testSuffix(t, "1.ip6.arpa.", "1.ip6.arpa.", true)
+	testSuffix(t, "www.some.arpa.", "some.arpa.", true)
+	testSuffix(t, "www.some.home.arpa.", "home.arpa.", true)
+	testSuffix(t, ".", "", false)
+	testSuffix(t, "", "", false)
+
+	// Test edge case domains.
+	testSuffix(t, "www.some.example.", "some.example.", true)
+	testSuffix(t, "www.some.invalid.", "some.invalid.", true)
+	testSuffix(t, "www.some.local.", "some.local.", true)
+	testSuffix(t, "www.some.localhost.", "some.localhost.", true)
+	testSuffix(t, "www.some.onion.", "some.onion.", false)
+	testSuffix(t, "www.some.test.", "some.test.", true)
+}
+
+func testSuffix(t *testing.T, fqdn, domainRoot string, icannSpace bool) {
+	t.Helper()
+
+	q := &Query{FQDN: fqdn}
+	q.InitPublicSuffixData()
+	assert.Equal(t, domainRoot, q.DomainRoot)
+	assert.Equal(t, icannSpace, q.ICANNSpace)
+}

From 65f646aad02a081330242a9491c1af25cc8ec92e Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 24 May 2022 11:27:44 +0200
Subject: [PATCH 41/86] Improve wording on setting dns/noInsecureProtocols

---
 resolver/config.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resolver/config.go b/resolver/config.go
index b274ee23..ff17cb99 100644
--- a/resolver/config.go
+++ b/resolver/config.go
@@ -224,9 +224,9 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 	noMulticastDNS = status.SecurityLevelOption(CfgOptionNoMulticastDNSKey)
 
 	err = config.Register(&config.Option{
-		Name:           "Enforce Secure DNS",
+		Name:           "Use Secure Protocols Only",
 		Key:            CfgOptionNoInsecureProtocolsKey,
-		Description:    "Never resolve using insecure protocols, ie. plain DNS.",
+		Description:    "Never resolve using insecure protocols, ie. plain DNS. This may break certain local DNS services, which always use plain DNS.",
 		OptType:        config.OptTypeInt,
 		ExpertiseLevel: config.ExpertiseLevelExpert,
 		ReleaseLevel:   config.ReleaseLevelStable,

From 515f4686f7eb28db095f6fe676e3d110bd7d00d4 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 25 May 2022 13:57:31 +0200
Subject: [PATCH 42/86] Send notification instead of killing conflicting DNS
 service

---
 nameserver/conflict.go |  75 +++++++++++++++++++
 nameserver/module.go   |  85 +++++++++++++++++++--
 nameserver/takeover.go | 164 -----------------------------------------
 3 files changed, 154 insertions(+), 170 deletions(-)
 create mode 100644 nameserver/conflict.go
 delete mode 100644 nameserver/takeover.go

diff --git a/nameserver/conflict.go b/nameserver/conflict.go
new file mode 100644
index 00000000..e02e1fd5
--- /dev/null
+++ b/nameserver/conflict.go
@@ -0,0 +1,75 @@
+package nameserver
+
+import (
+	"net"
+	"os"
+
+	processInfo "github.com/shirou/gopsutil/process"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/network/packet"
+	"github.com/safing/portmaster/network/state"
+)
+
+var commonResolverIPs = []net.IP{
+	net.IPv4zero,
+	net.IPv4(127, 0, 0, 1),  // default
+	net.IPv4(127, 0, 0, 53), // some resolvers on Linux
+	net.IPv6zero,
+	net.IPv6loopback,
+}
+
+func findConflictingProcess(ip net.IP, port uint16) (conflictingProcess *processInfo.Process) {
+	// Evaluate which IPs to check.
+	var ipsToCheck []net.IP
+	if ip.Equal(net.IPv4zero) || ip.Equal(net.IPv6zero) {
+		ipsToCheck = commonResolverIPs
+	} else {
+		ipsToCheck = []net.IP{ip}
+	}
+
+	// Find the conflicting process.
+	var err error
+	for _, resolverIP := range ipsToCheck {
+		conflictingProcess, err = getListeningProcess(resolverIP, port)
+		switch {
+		case err != nil:
+			// Log the error and let the worker try again.
+			log.Warningf("nameserver: failed to find conflicting service: %s", err)
+		case conflictingProcess != nil:
+			// Conflicting service found.
+			return conflictingProcess
+		}
+	}
+
+	return nil
+}
+
+func getListeningProcess(resolverIP net.IP, resolverPort uint16) (*processInfo.Process, error) {
+	pid, _, err := state.Lookup(&packet.Info{
+		Inbound:  true,
+		Version:  0, // auto-detect
+		Protocol: packet.UDP,
+		Src:      nil, // do not record direction
+		SrcPort:  0,   // do not record direction
+		Dst:      resolverIP,
+		DstPort:  resolverPort,
+	}, true)
+	if err != nil {
+		// there may be nothing listening on :53
+		return nil, nil //nolint:nilerr // Treat lookup error as "not found".
+	}
+
+	// Ignore if it's us for some reason.
+	if pid == os.Getpid() {
+		return nil, nil
+	}
+
+	proc, err := processInfo.NewProcess(int32(pid))
+	if err != nil {
+		// Process may have disappeared already.
+		return nil, err
+	}
+
+	return proc, nil
+}
diff --git a/nameserver/module.go b/nameserver/module.go
index 1c1dc871..f5165585 100644
--- a/nameserver/module.go
+++ b/nameserver/module.go
@@ -13,6 +13,7 @@ import (
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
+	"github.com/safing/portbase/notifications"
 	"github.com/safing/portmaster/compat"
 	"github.com/safing/portmaster/firewall"
 	"github.com/safing/portmaster/netenv"
@@ -25,6 +26,9 @@ var (
 	stopListener1     func() error
 	stopListener2     func() error
 	stopListenersLock sync.Mutex
+
+	eventIDConflictingService = "nameserver:conflicting-service"
+	eventIDListenerFailed     = "nameserver:listener-failed"
 )
 
 func init() {
@@ -133,24 +137,93 @@ func startListener(ip net.IP, port uint16, first bool) {
 			return nil
 		}
 
+		// Resolve generic listener error, if primary listener.
+		if first {
+			module.Resolve(eventIDListenerFailed)
+		}
+
 		// Start listening.
 		log.Infof("nameserver: starting to listen on %s", dnsServer.Addr)
 		err := dnsServer.ListenAndServe()
 		if err != nil {
-			// check if we are shutting down
+			// Stop worker without error if we are shutting down.
 			if module.IsStopping() {
 				return nil
 			}
-			// is something blocking our port?
-			checkErr := checkForConflictingService(ip, port)
-			if checkErr != nil {
-				return checkErr
-			}
+			log.Warningf("nameserver: failed to listen on %s: %s", dnsServer.Addr, err)
+			handleListenError(err, ip, port, first)
 		}
 		return err
 	})
 }
 
+func handleListenError(err error, ip net.IP, port uint16, primaryListener bool) {
+	var n *notifications.Notification
+
+	// Create suffix for secondary listener
+	var secondaryEventIDSuffix string
+	if !primaryListener {
+		secondaryEventIDSuffix = "-secondary"
+	}
+
+	// Find a conflicting service.
+	cfProcess := findConflictingProcess(ip, port)
+	if cfProcess != nil {
+		// Report the conflicting process.
+
+		// Build conflicting process description.
+		var cfDescription string
+		cfName, err := cfProcess.Name()
+		if err == nil && cfName != "" {
+			cfDescription = cfName
+		}
+		cfExe, err := cfProcess.Exe()
+		if err == nil && cfDescription != "" {
+			if cfDescription != "" {
+				cfDescription += " (" + cfExe + ")"
+			} else {
+				cfDescription = cfName
+			}
+		}
+
+		// Notify user about conflicting service.
+		n = notifications.Notify(&notifications.Notification{
+			EventID: eventIDConflictingService + secondaryEventIDSuffix,
+			Type:    notifications.Error,
+			Title:   "Conflicting DNS Software",
+			Message: fmt.Sprintf(
+				"Restart Portmaster after you have deactivated or properly configured the conflicting software: %s",
+				cfDescription,
+			),
+			ShowOnSystem: true,
+			AvailableActions: []*notifications.Action{
+				{
+					Text:    "Open Docs",
+					Type:    notifications.ActionTypeOpenURL,
+					Payload: "https://docs.safing.io/portmaster/install/status/software-compatibility",
+				},
+			},
+		})
+	} else {
+		// If no conflict is found, report the error directly.
+		n = notifications.Notify(&notifications.Notification{
+			EventID: eventIDListenerFailed + secondaryEventIDSuffix,
+			Type:    notifications.Error,
+			Title:   "Secure DNS Error",
+			Message: fmt.Sprintf(
+				"The internal DNS server failed. Restart Portmaster to try again. Error: %s",
+				err,
+			),
+			ShowOnSystem: true,
+		})
+	}
+
+	// Attach error to module, if primary listener.
+	if primaryListener {
+		n.AttachToModule(module)
+	}
+}
+
 func stop() error {
 	stopListenersLock.Lock()
 	defer stopListenersLock.Unlock()
diff --git a/nameserver/takeover.go b/nameserver/takeover.go
deleted file mode 100644
index fa1b65af..00000000
--- a/nameserver/takeover.go
+++ /dev/null
@@ -1,164 +0,0 @@
-package nameserver
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"strconv"
-	"time"
-
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portbase/notifications"
-	"github.com/safing/portmaster/network/packet"
-	"github.com/safing/portmaster/network/state"
-)
-
-var (
-	commonResolverIPs = []net.IP{
-		net.IPv4zero,
-		net.IPv4(127, 0, 0, 1),  // default
-		net.IPv4(127, 0, 0, 53), // some resolvers on Linux
-		net.IPv6zero,
-		net.IPv6loopback,
-	}
-
-	// lastKilledPID holds the PID of the last killed conflicting service.
-	// It is only accessed by checkForConflictingService, which is only called by
-	// the nameserver worker.
-	lastKilledPID int
-)
-
-func checkForConflictingService(ip net.IP, port uint16) error {
-	// Evaluate which IPs to check.
-	var ipsToCheck []net.IP
-	if ip.Equal(net.IPv4zero) || ip.Equal(net.IPv6zero) {
-		ipsToCheck = commonResolverIPs
-	} else {
-		ipsToCheck = []net.IP{ip}
-	}
-
-	// Check if there is another resolver when need to take over.
-	var killed int
-	var killingFailed bool
-ipsToCheckLoop:
-	for _, resolverIP := range ipsToCheck {
-		pid, err := takeover(resolverIP, port)
-		switch {
-		case err != nil:
-			// Log the error and let the worker try again.
-			log.Infof("nameserver: failed to stop conflicting service: %s", err)
-			killingFailed = true
-			break ipsToCheckLoop
-		case pid != 0:
-			// Conflicting service identified and killed!
-			killed = pid
-			break ipsToCheckLoop
-		}
-	}
-
-	// Notify user of failed killing or repeated kill.
-	if killingFailed || (killed != 0 && killed == lastKilledPID) {
-		// Notify the user that we failed to kill something.
-		notifications.Notify(&notifications.Notification{
-			EventID:      "namserver:failed-to-kill-conflicting-service",
-			Type:         notifications.Error,
-			Title:        "Failed to Stop Conflicting DNS Client",
-			Message:      "The Portmaster failed to stop a conflicting DNS client to gain required system integration. If there is another DNS Client (Nameserver; Resolver) on this device, please disable it.",
-			ShowOnSystem: true,
-			AvailableActions: []*notifications.Action{
-				{
-					ID:   "ack",
-					Text: "OK",
-				},
-				{
-					Text:    "Open Docs",
-					Type:    notifications.ActionTypeOpenURL,
-					Payload: "https://docs.safing.io/portmaster/install/status/software-compatibility",
-				},
-			},
-		})
-		return nil
-	}
-
-	// Check if something was killed.
-	if killed == 0 {
-		return nil
-	}
-	lastKilledPID = killed
-
-	// Notify the user that we killed something.
-	notifications.Notify(&notifications.Notification{
-		EventID: "namserver:stopped-conflicting-service",
-		Type:    notifications.Info,
-		Title:   "Stopped Conflicting DNS Client",
-		Message: fmt.Sprintf(
-			"The Portmaster stopped a conflicting DNS client (pid %d) to gain required system integration. If you are running another DNS client on this device on purpose, you can the check the documentation if it is compatible with the Portmaster.",
-			killed,
-		),
-		ShowOnSystem: true,
-		AvailableActions: []*notifications.Action{
-			{
-				ID:   "ack",
-				Text: "OK",
-			},
-			{
-				Text:    "Open Docs",
-				Type:    notifications.ActionTypeOpenURL,
-				Payload: "https://docs.safing.io/portmaster/install/status/software-compatibility",
-			},
-		},
-	})
-
-	// Restart nameserver via service-worker logic.
-	// Wait shortly so that the other process can shut down.
-	time.Sleep(10 * time.Millisecond)
-	return fmt.Errorf("%w: stopped conflicting name service with pid %d", modules.ErrRestartNow, killed)
-}
-
-func takeover(resolverIP net.IP, resolverPort uint16) (int, error) {
-	pid, _, err := state.Lookup(&packet.Info{
-		Inbound:  true,
-		Version:  0, // auto-detect
-		Protocol: packet.UDP,
-		Src:      nil, // do not record direction
-		SrcPort:  0,   // do not record direction
-		Dst:      resolverIP,
-		DstPort:  resolverPort,
-	}, true)
-	if err != nil {
-		// there may be nothing listening on :53
-		return 0, nil //nolint:nilerr // Treat lookup error as "not found".
-	}
-
-	// Just don't, uh, kill ourselves...
-	if pid == os.Getpid() {
-		return 0, nil
-	}
-
-	proc, err := os.FindProcess(pid)
-	if err != nil {
-		// huh. gone already? I guess we'll wait then...
-		return 0, err
-	}
-
-	err = proc.Signal(os.Interrupt)
-	if err != nil {
-		err = proc.Kill()
-		if err != nil {
-			log.Errorf("nameserver: failed to stop conflicting service (pid %d): %s", pid, err)
-			return 0, err
-		}
-	}
-
-	log.Warningf(
-		"nameserver: killed conflicting service with PID %d over %s",
-		pid,
-		net.JoinHostPort(
-			resolverIP.String(),
-			strconv.Itoa(int(resolverPort)),
-		),
-	)
-
-	return pid, nil
-}

From 787f9e7dec106e79f36196caa8f1047ee7e7c8cc Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 1 Jun 2022 13:28:10 +0200
Subject: [PATCH 43/86] Add support for upcoming UNBREAK filter list

---
 firewall/filter.go | 18 +++++++++++++++++-
 firewall/master.go | 10 ++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/firewall/filter.go b/firewall/filter.go
index 58a24737..390ef3f3 100644
--- a/firewall/filter.go
+++ b/firewall/filter.go
@@ -2,9 +2,11 @@ package firewall
 
 import (
 	"github.com/safing/portbase/config"
+	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
 	_ "github.com/safing/portmaster/core"
+	"github.com/safing/portmaster/intel/filterlists"
 	"github.com/safing/spn/captain"
 )
 
@@ -12,10 +14,13 @@ var (
 	filterModule  *modules.Module
 	filterEnabled config.BoolOption
 	tunnelEnabled config.BoolOption
+
+	unbreakFilterListIDs         = []string{"UNBREAK"}
+	resolvedUnbreakFilterListIDs []string
 )
 
 func init() {
-	filterModule = modules.Register("filter", filterPrep, nil, nil, "core", "intel")
+	filterModule = modules.Register("filter", filterPrep, filterStart, nil, "core", "intel")
 	subsystems.Register(
 		"filter",
 		"Privacy Filter",
@@ -47,3 +52,14 @@ func filterPrep() (err error) {
 	tunnelEnabled = config.Concurrent.GetAsBool(captain.CfgOptionEnableSPNKey, false)
 	return nil
 }
+
+func filterStart() error {
+	// TODO: Re-resolve IDs when filterlist index changes.
+	resolvedIDs, err := filterlists.ResolveListIDs(unbreakFilterListIDs)
+	if err != nil {
+		log.Warningf("filter: failed to resolve unbreak filter list IDs: %s", err)
+	} else {
+		resolvedUnbreakFilterListIDs = resolvedIDs
+	}
+	return nil
+}
diff --git a/firewall/master.go b/firewall/master.go
index e50617d0..1de7f194 100644
--- a/firewall/master.go
+++ b/firewall/master.go
@@ -432,6 +432,16 @@ func checkFilterLists(ctx context.Context, conn *network.Connection, p *profile.
 	result, reason := p.MatchFilterLists(ctx, conn.Entity)
 	switch result {
 	case endpoints.Denied:
+		// If the connection matches a filter list, check if the "unbreak" list matches too and abort blocking.
+		for _, blockedListID := range conn.Entity.BlockedByLists {
+			for _, unbreakListID := range resolvedUnbreakFilterListIDs {
+				if blockedListID == unbreakListID {
+					log.Tracer(ctx).Debugf("filter: unbreak filter %s matched, ignoring other filter list matches", unbreakListID)
+					return false
+				}
+			}
+		}
+		// Otherwise, continue with blocking.
 		conn.DenyWithContext(reason.String(), profile.CfgOptionFilterListsKey, reason.Context())
 		return true
 	case endpoints.NoMatch:

From 911a80996a6abc208fe078c3461ab40d6bd3a3a3 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 1 Jun 2022 13:45:13 +0200
Subject: [PATCH 44/86] Fix multicast detection on ended connections as well as
 some linter errors

---
 core/base/databases.go    | 2 --
 network/multicast.go      | 4 ++--
 profile/module.go         | 2 --
 resolver/resolver_test.go | 3 +--
 4 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/core/base/databases.go b/core/base/databases.go
index d2ba2552..56355125 100644
--- a/core/base/databases.go
+++ b/core/base/databases.go
@@ -2,8 +2,6 @@ package base
 
 import (
 	"github.com/safing/portbase/database"
-
-	// Dependencies.
 	_ "github.com/safing/portbase/database/dbmodule"
 	_ "github.com/safing/portbase/database/storage/bbolt"
 )
diff --git a/network/multicast.go b/network/multicast.go
index 82cb2173..8a32aea7 100644
--- a/network/multicast.go
+++ b/network/multicast.go
@@ -17,8 +17,8 @@ func GetMulticastRequestConn(responseConn *Connection, responseFromNet *net.IPNe
 		switch {
 		case conn.Inbound:
 			// Ignore incoming connections.
-		//case conn.Ended != 0:
-		// Ignore ended connections.
+		case conn.Ended != 0:
+			// Ignore ended connections.
 		case conn.Entity.Protocol != responseConn.Entity.Protocol:
 			// Ignore on protocol mismatch.
 		case conn.LocalPort != responseConn.LocalPort:
diff --git a/profile/module.go b/profile/module.go
index da479663..31b52704 100644
--- a/profile/module.go
+++ b/profile/module.go
@@ -6,8 +6,6 @@ import (
 	"github.com/safing/portbase/database/migration"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
-
-	// Dependency.
 	_ "github.com/safing/portmaster/core/base"
 	"github.com/safing/portmaster/updates"
 )
diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go
index 61e05376..a03b3eb9 100644
--- a/resolver/resolver_test.go
+++ b/resolver/resolver_test.go
@@ -7,9 +7,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/safing/portbase/log"
 )

From 8c146488ac67bf13f768b14cb9fac73b504ae192 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 1 Jun 2022 13:53:59 +0200
Subject: [PATCH 45/86] Update SPN

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 43e69e3c..997bdcb0 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.49
 	github.com/oschwald/maxminddb-golang v1.9.0
 	github.com/safing/portbase v0.14.4
-	github.com/safing/spn v0.4.10
+	github.com/safing/spn v0.4.11
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
diff --git a/go.sum b/go.sum
index 4b2d7c17..7fe99705 100644
--- a/go.sum
+++ b/go.sum
@@ -860,6 +860,8 @@ github.com/safing/spn v0.4.9 h1:p+TRS2y1yAY4CYyHiVuy+V3NcA9p826NtYksuBaSl64=
 github.com/safing/spn v0.4.9/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.10 h1:nZuWPBn1z0aBUDnnpG9pKZCmPAH2hgNE4cqiSB7JGU8=
 github.com/safing/spn v0.4.10/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
+github.com/safing/spn v0.4.11 h1:Er3ZtBCqKaf0G6lmRKpm1LenlrWeaWUp8612bpeG+aM=
+github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

From f135ec3242905b84811b26975a3458117cc2c28e Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 1 Jun 2022 13:54:06 +0200
Subject: [PATCH 46/86] Bump core version

---
 cmds/portmaster-core/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index 3b4e7b04..b7f1d1ec 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.12", "AGPLv3", true)
+	info.Set("Portmaster", "0.8.13", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")

From c9b9a0b1d19506d6b985e8b58ba182e2601410b6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 9 Jun 2022 13:48:55 +0200
Subject: [PATCH 47/86] Improve read rights on log files

---
 cmds/portmaster-start/logs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmds/portmaster-start/logs.go b/cmds/portmaster-start/logs.go
index d14f1250..09fa01c6 100644
--- a/cmds/portmaster-start/logs.go
+++ b/cmds/portmaster-start/logs.go
@@ -17,7 +17,7 @@ import (
 )
 
 func initializeLogFile(logFilePath string, identifier string, version string) *os.File {
-	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0o0444)
+	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0o0440)
 	if err != nil {
 		log.Printf("failed to create log file %s: %s\n", logFilePath, err)
 		return nil

From 0439894efc3097214178a31e70718f66b81c4c09 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 9 Jun 2022 13:49:34 +0200
Subject: [PATCH 48/86] Suppress error when stopping during filterlist update

---
 intel/filterlists/updater.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/intel/filterlists/updater.go b/intel/filterlists/updater.go
index ff5ce3e9..7d15e85e 100644
--- a/intel/filterlists/updater.go
+++ b/intel/filterlists/updater.go
@@ -24,6 +24,11 @@ var updateInProgress = abool.New()
 func tryListUpdate(ctx context.Context) error {
 	err := performUpdate(ctx)
 	if err != nil {
+		// Check if we are shutting down.
+		if module.IsStopping() {
+			return nil
+		}
+
 		// Check if the module already has a failure status set. If not, set a
 		// generic one with the returned error.
 		failureStatus, _, _ := module.FailureStatus()

From c442a7e51c1c605a6f016ebe9a7591ffb5840e73 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 9 Jun 2022 13:50:18 +0200
Subject: [PATCH 49/86] Deactivate IPv6 integrations when no IPv6 stack is
 detected

---
 firewall/interception/nfqueue_linux.go | 52 ++++++++++++++++----------
 nameserver/module.go                   |  6 ++-
 netenv/main.go                         | 23 ++++++++++++
 resolver/resolver-mdns.go              | 51 +++++++++++++------------
 4 files changed, 89 insertions(+), 43 deletions(-)

diff --git a/firewall/interception/nfqueue_linux.go b/firewall/interception/nfqueue_linux.go
index f117cd67..9d731648 100644
--- a/firewall/interception/nfqueue_linux.go
+++ b/firewall/interception/nfqueue_linux.go
@@ -10,8 +10,8 @@ import (
 	"github.com/hashicorp/go-multierror"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/notifications"
 	"github.com/safing/portmaster/firewall/interception/nfq"
+	"github.com/safing/portmaster/netenv"
 	"github.com/safing/portmaster/network/packet"
 )
 
@@ -141,13 +141,10 @@ func activateNfqueueFirewall() error {
 		return err
 	}
 
-	if err := activateIPTables(iptables.ProtocolIPv6, v6rules, v6once, v6chains); err != nil {
-		notifications.NotifyError(
-			"interception:ipv6-possibly-disabled",
-			"Is IPv6 enabled?",
-			"The Portmaster succeeded with IPv4 network integration, but failed with IPv6 integration. Please make sure IPv6 is enabled on your device.",
-		)
-		return err
+	if netenv.IPv6Enabled() {
+		if err := activateIPTables(iptables.ProtocolIPv6, v6rules, v6once, v6chains); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -163,8 +160,10 @@ func DeactivateNfqueueFirewall() error {
 	}
 
 	// IPv6
-	if err := deactivateIPTables(iptables.ProtocolIPv6, v6once, v6chains); err != nil {
-		result = multierror.Append(result, err)
+	if netenv.IPv6Enabled() {
+		if err := deactivateIPTables(iptables.ProtocolIPv6, v6once, v6chains); err != nil {
+			result = multierror.Append(result, err)
+		}
 	}
 
 	return result.ErrorOrNil()
@@ -264,15 +263,22 @@ func StartNfqueueInterception(packets chan<- packet.Packet) (err error) {
 		_ = Stop()
 		return fmt.Errorf("nfqueue(IPv4, in): %w", err)
 	}
-	out6Queue, err = nfq.New(17060, true)
-	if err != nil {
-		_ = Stop()
-		return fmt.Errorf("nfqueue(IPv6, out): %w", err)
-	}
-	in6Queue, err = nfq.New(17160, true)
-	if err != nil {
-		_ = Stop()
-		return fmt.Errorf("nfqueue(IPv6, in): %w", err)
+
+	if netenv.IPv6Enabled() {
+		out6Queue, err = nfq.New(17060, true)
+		if err != nil {
+			_ = Stop()
+			return fmt.Errorf("nfqueue(IPv6, out): %w", err)
+		}
+		in6Queue, err = nfq.New(17160, true)
+		if err != nil {
+			_ = Stop()
+			return fmt.Errorf("nfqueue(IPv6, in): %w", err)
+		}
+	} else {
+		log.Warningf("interception: no IPv6 stack detected, disabling IPv6 network integration")
+		out6Queue = &disabledNfQueue{}
+		in6Queue = &disabledNfQueue{}
 	}
 
 	go handleInterception(packets)
@@ -327,3 +333,11 @@ func handleInterception(packets chan<- packet.Packet) {
 		}
 	}
 }
+
+type disabledNfQueue struct{}
+
+func (dnfq *disabledNfQueue) PacketChannel() <-chan packet.Packet {
+	return nil
+}
+
+func (dnfq *disabledNfQueue) Destroy() {}
diff --git a/nameserver/module.go b/nameserver/module.go
index f5165585..ed7eb740 100644
--- a/nameserver/module.go
+++ b/nameserver/module.go
@@ -259,7 +259,11 @@ func getListenAddresses(listenAddress string) (ip1, ip2 net.IP, port uint16, err
 	// listen separately for IPv4 and IPv6.
 	if ipString == "localhost" {
 		ip1 = net.IPv4(127, 0, 0, 17)
-		ip2 = net.IPv6loopback
+		if netenv.IPv6Enabled() {
+			ip2 = net.IPv6loopback
+		} else {
+			log.Warningf("nameserver: no IPv6 stack detected, disabling IPv6 nameserver listener")
+		}
 	} else {
 		ip1 = net.ParseIP(ipString)
 		if ip1 == nil {
diff --git a/netenv/main.go b/netenv/main.go
index 0d831e76..232696cd 100644
--- a/netenv/main.go
+++ b/netenv/main.go
@@ -1,7 +1,9 @@
 package netenv
 
 import (
+	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
+	"github.com/tevino/abool"
 )
 
 // Event Names.
@@ -20,6 +22,8 @@ func init() {
 }
 
 func prep() error {
+	checkForIPv6Stack()
+
 	if err := registerAPIEndpoints(); err != nil {
 		return err
 	}
@@ -46,3 +50,22 @@ func start() error {
 
 	return nil
 }
+
+var ipv6Enabled = abool.NewBool(true)
+
+// IPv6Enabled returns whether the device has an active IPv6 stack.
+// This is only checked once on startup in order to maintain consistency.
+func IPv6Enabled() bool {
+	return ipv6Enabled.IsSet()
+}
+
+func checkForIPv6Stack() {
+	_, v6IPs, err := GetAssignedAddresses()
+	if err != nil {
+		log.Warningf("netenv: failed to get assigned addresses to check for ipv6 stack: %s", err)
+		return
+	}
+
+	// Set IPv6 as enabled if any IPv6 addresses are found.
+	ipv6Enabled.SetTo(len(v6IPs) > 0)
+}
diff --git a/resolver/resolver-mdns.go b/resolver/resolver-mdns.go
index ce0eab6d..29677350 100644
--- a/resolver/resolver-mdns.go
+++ b/resolver/resolver-mdns.go
@@ -12,6 +12,7 @@ import (
 	"github.com/miekg/dns"
 
 	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/netenv"
 	"github.com/safing/portmaster/network/netutils"
 )
 
@@ -91,19 +92,6 @@ func listenToMDNS(ctx context.Context) error {
 		}()
 	}
 
-	multicast6Conn, err = net.ListenMulticastUDP("udp6", nil, &net.UDPAddr{IP: net.IP([]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb}), Port: 5353})
-	if err != nil {
-		// TODO: retry after some time
-		log.Warningf("intel(mdns): failed to create udp6 listen multicast socket: %s", err)
-	} else {
-		module.StartServiceWorker("mdns udp6 multicast listener", 0, func(ctx context.Context) error {
-			return listenForDNSPackets(ctx, multicast6Conn, messages)
-		})
-		defer func() {
-			_ = multicast6Conn.Close()
-		}()
-	}
-
 	unicast4Conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
 	if err != nil {
 		// TODO: retry after some time
@@ -117,17 +105,34 @@ func listenToMDNS(ctx context.Context) error {
 		}()
 	}
 
-	unicast6Conn, err = net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6zero, Port: 0})
-	if err != nil {
-		// TODO: retry after some time
-		log.Warningf("intel(mdns): failed to create udp6 listen socket: %s", err)
+	if netenv.IPv6Enabled() {
+		multicast6Conn, err = net.ListenMulticastUDP("udp6", nil, &net.UDPAddr{IP: net.IP([]byte{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb}), Port: 5353})
+		if err != nil {
+			// TODO: retry after some time
+			log.Warningf("intel(mdns): failed to create udp6 listen multicast socket: %s", err)
+		} else {
+			module.StartServiceWorker("mdns udp6 multicast listener", 0, func(ctx context.Context) error {
+				return listenForDNSPackets(ctx, multicast6Conn, messages)
+			})
+			defer func() {
+				_ = multicast6Conn.Close()
+			}()
+		}
+
+		unicast6Conn, err = net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6zero, Port: 0})
+		if err != nil {
+			// TODO: retry after some time
+			log.Warningf("intel(mdns): failed to create udp6 listen socket: %s", err)
+		} else {
+			module.StartServiceWorker("mdns udp6 unicast listener", 0, func(ctx context.Context) error {
+				return listenForDNSPackets(ctx, unicast6Conn, messages)
+			})
+			defer func() {
+				_ = unicast6Conn.Close()
+			}()
+		}
 	} else {
-		module.StartServiceWorker("mdns udp6 unicast listener", 0, func(ctx context.Context) error {
-			return listenForDNSPackets(ctx, unicast6Conn, messages)
-		})
-		defer func() {
-			_ = unicast6Conn.Close()
-		}()
+		log.Warningf("resolver: no IPv6 stack detected, disabling IPv6 mDNS resolver")
 	}
 
 	// start message handler

From b392a1e8fff5a6dae3a1f1619a1865fc7b24f0cd Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 9 Jun 2022 13:59:50 +0200
Subject: [PATCH 50/86] Add geoip.IsInitialized to expose if the databases have
 been loaded

---
 intel/geoip/lookup.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/intel/geoip/lookup.go b/intel/geoip/lookup.go
index 7210bc79..c635b574 100644
--- a/intel/geoip/lookup.go
+++ b/intel/geoip/lookup.go
@@ -26,3 +26,8 @@ func GetLocation(ip net.IP) (*Location, error) {
 	record.FillMissingInfo()
 	return record, nil
 }
+
+// IsInitialized returns whether the geoip database has been initialized.
+func IsInitialized(v6, wait bool) bool {
+	return worker.GetReader(v6, wait) != nil
+}

From 86d4f64d429ea37e43d899dc5b12f21a2553609a Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 9 Jun 2022 14:36:03 +0200
Subject: [PATCH 51/86] Improve device location system with more safeguards

---
 netenv/api.go      |   2 +-
 netenv/location.go | 151 ++++++++++++++++++++++++---------------------
 2 files changed, 80 insertions(+), 73 deletions(-)

diff --git a/netenv/api.go b/netenv/api.go
index 237dee57..20a2f688 100644
--- a/netenv/api.go
+++ b/netenv/api.go
@@ -55,7 +55,7 @@ func registerAPIEndpoints() error {
 		Read:      api.PermitUser,
 		BelongsTo: module,
 		StructFunc: func(ar *api.Request) (i interface{}, err error) {
-			return getLocationFromTraceroute()
+			return getLocationFromTraceroute(&DeviceLocations{})
 		},
 		Name:        "Get Approximate Internet Location via Traceroute",
 		Description: "Returns an approximation of where the device is on the Internet using a the traceroute technique.",
diff --git a/netenv/location.go b/netenv/location.go
index 6245ab45..8c8c5d56 100644
--- a/netenv/location.go
+++ b/netenv/location.go
@@ -47,16 +47,16 @@ type DeviceLocations struct {
 }
 
 // Best returns the best (most accurate) device location.
-func (dl *DeviceLocations) Best() *DeviceLocation {
-	if len(dl.All) > 0 {
-		return dl.All[0]
+func (dls *DeviceLocations) Best() *DeviceLocation {
+	if len(dls.All) > 0 {
+		return dls.All[0]
 	}
 	return nil
 }
 
 // BestV4 returns the best (most accurate) IPv4 device location.
-func (dl *DeviceLocations) BestV4() *DeviceLocation {
-	for _, loc := range dl.All {
+func (dls *DeviceLocations) BestV4() *DeviceLocation {
+	for _, loc := range dls.All {
 		if loc.IPVersion == packet.IPv4 {
 			return loc
 		}
@@ -65,8 +65,8 @@ func (dl *DeviceLocations) BestV4() *DeviceLocation {
 }
 
 // BestV6 returns the best (most accurate) IPv6 device location.
-func (dl *DeviceLocations) BestV6() *DeviceLocation {
-	for _, loc := range dl.All {
+func (dls *DeviceLocations) BestV6() *DeviceLocation {
+	for _, loc := range dls.All {
 		if loc.IPVersion == packet.IPv6 {
 			return loc
 		}
@@ -74,11 +74,8 @@ func (dl *DeviceLocations) BestV6() *DeviceLocation {
 	return nil
 }
 
-func copyDeviceLocations() *DeviceLocations {
-	locationsLock.Lock()
-	defer locationsLock.Unlock()
-
-	// Create a copy of the locations, but not the entries.
+// Copy creates a copy of the locations, but not the individual entries.
+func (dls *DeviceLocations) Copy() *DeviceLocations {
 	cp := &DeviceLocations{
 		All: make([]*DeviceLocation, len(locations.All)),
 	}
@@ -87,6 +84,32 @@ func copyDeviceLocations() *DeviceLocations {
 	return cp
 }
 
+// AddLocation adds a location.
+func (dls *DeviceLocations) AddLocation(dl *DeviceLocation) {
+	if dls == nil {
+		return
+	}
+
+	// Add to locations, if better.
+	var exists bool
+	for i, existing := range dls.All {
+		if (dl.IP == nil && existing.IP == nil) || dl.IP.Equal(existing.IP) {
+			exists = true
+			if dl.IsMoreAccurateThan(existing) {
+				// Replace
+				dls.All[i] = dl
+				break
+			}
+		}
+	}
+	if !exists {
+		dls.All = append(dls.All, dl)
+	}
+
+	// Sort locations.
+	sort.Sort(sortLocationsByAccuracy(dls.All))
+}
+
 // DeviceLocation represents a single IP and metadata. It must not be changed
 // once created.
 type DeviceLocation struct {
@@ -147,6 +170,12 @@ func (dl *DeviceLocation) String() string {
 		return "<none>"
 	case dl.Location == nil:
 		return dl.IP.String()
+	case dl.Source == SourceTimezone:
+		return fmt.Sprintf(
+			"TZ(%.0f/%.0f)",
+			dl.Location.Coordinates.Latitude,
+			dl.Location.Coordinates.Longitude,
+		)
 	default:
 		return fmt.Sprintf("%s (AS%d in %s)", dl.IP, dl.Location.AutonomousSystemNumber, dl.Location.Country.ISOCode)
 	}
@@ -193,6 +222,14 @@ func (a sortLocationsByAccuracy) Less(i, j int) bool { return !a[j].IsMoreAccura
 
 // SetInternetLocation provides the location management system with a possible Internet location.
 func SetInternetLocation(ip net.IP, source DeviceLocationSource) (dl *DeviceLocation, ok bool) {
+	locationsLock.Lock()
+	defer locationsLock.Unlock()
+
+	return locations.AddIP(ip, source)
+}
+
+// AddIP adds a new location based on the given IP.
+func (dls *DeviceLocations) AddIP(ip net.IP, source DeviceLocationSource) (dl *DeviceLocation, ok bool) {
 	// Check if IP is global.
 	if netutils.GetIPScope(ip) != netutils.Global {
 		return nil, false
@@ -222,38 +259,10 @@ func SetInternetLocation(ip net.IP, source DeviceLocationSource) (dl *DeviceLoca
 	}
 	loc.Location = geoLoc
 
-	addLocation(loc)
+	dls.AddLocation(loc)
 	return loc, true
 }
 
-func addLocation(dl *DeviceLocation) {
-	if dl == nil {
-		return
-	}
-
-	locationsLock.Lock()
-	defer locationsLock.Unlock()
-
-	// Add to locations, if better.
-	var exists bool
-	for i, existing := range locations.All {
-		if (dl.IP == nil && existing.IP == nil) || dl.IP.Equal(existing.IP) {
-			exists = true
-			if dl.IsMoreAccurateThan(existing) {
-				// Replace
-				locations.All[i] = dl
-				break
-			}
-		}
-	}
-	if !exists {
-		locations.All = append(locations.All, dl)
-	}
-
-	// Sort locations.
-	sort.Sort(sortLocationsByAccuracy(locations.All))
-}
-
 // GetApproximateInternetLocation returns the approximate Internet location.
 // Deprecated: Please use GetInternetLocation instead.
 func GetApproximateInternetLocation() (net.IP, error) {
@@ -271,30 +280,21 @@ func GetInternetLocation() (deviceLocations *DeviceLocations, ok bool) {
 
 	// Check if the network changed, if not, return cache.
 	if !locationNetworkChangedFlag.IsSet() {
-		return copyDeviceLocations(), true
+		locationsLock.Lock()
+		defer locationsLock.Unlock()
+		return locations.Copy(), true
 	}
 	locationNetworkChangedFlag.Refresh()
 
-	// Reset locations.
-	func() {
-		locationsLock.Lock()
-		defer locationsLock.Unlock()
-		locations = &DeviceLocations{}
-	}()
-
-	// Get all assigned addresses.
-	v4s, v6s, err := GetAssignedAddresses()
-	if err != nil {
-		log.Warningf("netenv: failed to get assigned addresses for device location: %s", err)
-		return nil, false
-	}
+	// Create new location list.
+	dls := &DeviceLocations{}
 
 	// Check interfaces for global addresses.
-	v4ok, v6ok := getLocationFromInterfaces()
+	v4ok, v6ok := getLocationFromInterfaces(dls)
 
 	// Try other methods for missing locations.
-	if len(v4s) > 0 && !v4ok {
-		_, err = getLocationFromTraceroute()
+	if !v4ok {
+		_, err := getLocationFromTraceroute(dls)
 		if err != nil {
 			log.Warningf("netenv: failed to get IPv4 device location from traceroute: %s", err)
 		} else {
@@ -303,35 +303,43 @@ func GetInternetLocation() (deviceLocations *DeviceLocations, ok bool) {
 
 		// Get location from timezone as final fallback.
 		if !v4ok {
-			getLocationFromTimezone(packet.IPv4)
+			getLocationFromTimezone(dls, packet.IPv4)
 		}
 	}
-	if len(v6s) > 0 && !v6ok {
+	if !v6ok && IPv6Enabled() {
 		// TODO: Find more ways to get IPv6 device location
 
 		// Get location from timezone as final fallback.
-		getLocationFromTimezone(packet.IPv6)
+		getLocationFromTimezone(dls, packet.IPv6)
 	}
 
+	// As a last guard, make sure there is at least one location in the list.
+	if len(dls.All) == 0 {
+		getLocationFromTimezone(dls, packet.IPv4)
+	}
+
+	// Set new locations.
+	locationsLock.Lock()
+	defer locationsLock.Unlock()
+	locations = dls
+
 	// Return gathered locations.
-	cp := copyDeviceLocations()
-	return cp, true
+	return locations.Copy(), true
 }
 
-func getLocationFromInterfaces() (v4ok, v6ok bool) {
+func getLocationFromInterfaces(dls *DeviceLocations) (v4ok, v6ok bool) {
 	globalIPv4, globalIPv6, err := GetAssignedGlobalAddresses()
 	if err != nil {
 		log.Warningf("netenv: location: failed to get assigned global addresses: %s", err)
 		return false, false
 	}
-
 	for _, ip := range globalIPv4 {
-		if _, ok := SetInternetLocation(ip, SourceInterface); ok {
+		if _, ok := dls.AddIP(ip, SourceInterface); ok {
 			v4ok = true
 		}
 	}
 	for _, ip := range globalIPv6 {
-		if _, ok := SetInternetLocation(ip, SourceInterface); ok {
+		if _, ok := dls.AddIP(ip, SourceInterface); ok {
 			v6ok = true
 		}
 	}
@@ -349,7 +357,7 @@ func getLocationFromUPnP() (ok bool) {
 }
 */
 
-func getLocationFromTraceroute() (dl *DeviceLocation, err error) {
+func getLocationFromTraceroute(dls *DeviceLocations) (dl *DeviceLocation, err error) {
 	// Create connection.
 	conn, err := net.ListenPacket("ip4:icmp", "")
 	if err != nil {
@@ -470,7 +478,7 @@ nextHop:
 					// We have received a valid time exceeded error.
 					// If message came from a global unicast, us it!
 					if netutils.GetIPScope(remoteIP) == netutils.Global {
-						dl, ok := SetInternetLocation(remoteIP, SourceTraceroute)
+						dl, ok := dls.AddIP(remoteIP, SourceTraceroute)
 						if !ok {
 							return nil, errors.New("invalid IP address")
 						}
@@ -516,7 +524,7 @@ func recvICMP(currentHop int, icmpPacketsViaFirewall chan packet.Packet) (
 	}
 }
 
-func getLocationFromTimezone(ipVersion packet.IPVersion) (ok bool) { //nolint:unparam // This is documentation.
+func getLocationFromTimezone(dls *DeviceLocations, ipVersion packet.IPVersion) {
 	// Create base struct.
 	tzLoc := &DeviceLocation{
 		IPVersion:      ipVersion,
@@ -531,6 +539,5 @@ func getLocationFromTimezone(ipVersion packet.IPVersion) (ok bool) { //nolint:un
 	tzLoc.Location.Coordinates.Latitude = 48
 	tzLoc.Location.Coordinates.Longitude = float64(offsetSeconds) / 43200 * 180
 
-	addLocation(tzLoc)
-	return true
+	dls.AddLocation(tzLoc)
 }

From 62ec170b90ae5655db04459c4b07d4f068826ac1 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Wed, 16 Mar 2022 20:37:57 +0100
Subject: [PATCH 52/86] Add basic, tailored SQL ORM mapper

---
 go.mod                              |  80 +----
 go.sum                              | 139 ++++++++
 netquery/orm/decoder.go             | 404 +++++++++++++++++++++++
 netquery/orm/decoder_test.go        | 475 ++++++++++++++++++++++++++++
 netquery/orm/encoder.go             | 124 ++++++++
 netquery/orm/encoder_test.go        | 126 ++++++++
 netquery/orm/query_runner.go        | 123 +++++++
 netquery/orm/schema_builder.go      | 220 +++++++++++++
 netquery/orm/schema_builder_test.go |  41 +++
 9 files changed, 1669 insertions(+), 63 deletions(-)
 create mode 100644 netquery/orm/decoder.go
 create mode 100644 netquery/orm/decoder_test.go
 create mode 100644 netquery/orm/encoder.go
 create mode 100644 netquery/orm/encoder_test.go
 create mode 100644 netquery/orm/query_runner.go
 create mode 100644 netquery/orm/schema_builder.go
 create mode 100644 netquery/orm/schema_builder_test.go

diff --git a/go.mod b/go.mod
index 997bdcb0..3100d1f9 100644
--- a/go.mod
+++ b/go.mod
@@ -1,80 +1,34 @@
 module github.com/safing/portmaster
 
-go 1.18
+go 1.15
 
 require (
 	github.com/agext/levenshtein v1.2.3
 	github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d
 	github.com/coreos/go-iptables v0.6.0
-	github.com/florianl/go-nfqueue v1.3.1
+	github.com/florianl/go-nfqueue v1.3.0
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.4.0
-	github.com/miekg/dns v1.1.49
-	github.com/oschwald/maxminddb-golang v1.9.0
-	github.com/safing/portbase v0.14.4
-	github.com/safing/spn v0.4.11
+	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/mdlayher/socket v0.2.2 // indirect
+	github.com/miekg/dns v1.1.46
+	github.com/oschwald/maxminddb-golang v1.8.0
+	github.com/safing/portbase v0.14.0
+	github.com/safing/spn v0.4.3
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/spf13/cobra v1.4.0
-	github.com/stretchr/testify v1.7.1
+	github.com/spf13/cobra v1.3.0
+	github.com/stretchr/testify v1.7.0
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220513224357-95641704303c
-	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
-	golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
+	zombiezen.com/go/sqlite v0.9.2
 )
 
-require (
-	github.com/VictoriaMetrics/metrics v1.18.1 // indirect
-	github.com/aead/ecdh v0.2.0 // indirect
-	github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 // indirect
-	github.com/armon/go-radix v1.0.0 // indirect
-	github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect
-	github.com/bluele/gcache v0.0.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/gofrs/uuid v4.2.0+incompatible // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/josharian/native v1.0.0 // indirect
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/mdlayher/netlink v1.6.0 // indirect
-	github.com/mdlayher/socket v0.2.3 // indirect
-	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/mr-tron/base58 v1.2.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rot256/pblind v0.0.0-20211117203330-22455f90b565 // indirect
-	github.com/safing/jess v0.2.3 // indirect
-	github.com/satori/go.uuid v1.2.0 // indirect
-	github.com/seehuhn/fortuna v1.0.1 // indirect
-	github.com/seehuhn/sha256d v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tidwall/gjson v1.14.1 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/sjson v1.2.4 // indirect
-	github.com/tklauser/go-sysconf v0.3.10 // indirect
-	github.com/tklauser/numcpus v0.5.0 // indirect
-	github.com/valyala/fastrand v1.1.0 // indirect
-	github.com/valyala/histogram v1.2.0 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
-	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/x448/float16 v0.8.4 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-)
+replace github.com/safing/spn => ../spn
+
+replace github.com/safing/portbase => ../portbase
diff --git a/go.sum b/go.sum
index 7fe99705..390453ed 100644
--- a/go.sum
+++ b/go.sum
@@ -66,6 +66,7 @@ cloud.google.com/go/trace v0.1.0/go.mod h1:wxEwsoeRVPbeSkt7ZC9nWCgmoKQRAoySN7XHW
 contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
 contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ=
 contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
+crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AlecAivazis/survey/v2 v2.0.7/go.mod h1:mlizQTaPjnR4jcpwRSaSlkbsRfYFEyKgLQvYTzxxiHA=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
@@ -459,6 +460,7 @@ github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -473,6 +475,7 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -646,7 +649,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
 github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
@@ -788,6 +793,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -969,6 +976,7 @@ github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp1
 github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
 github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
 github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
 github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
@@ -1082,6 +1090,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -1131,6 +1140,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
@@ -1314,6 +1324,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1350,6 +1361,7 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1455,6 +1467,7 @@ golang.org/x/tools v0.0.0-20200808161706-5bf02b21f123/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
@@ -1466,6 +1479,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
@@ -1696,6 +1710,129 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
+modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
+modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
+modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag=
+modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw=
+modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ=
+modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c=
+modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo=
+modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg=
+modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I=
+modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs=
+modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8=
+modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE=
+modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk=
+modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w=
+modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE=
+modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8=
+modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc=
+modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU=
+modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE=
+modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk=
+modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI=
+modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE=
+modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg=
+modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74=
+modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU=
+modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
+modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
+modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
+modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
+modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
+modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
+modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY=
+modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
+modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU=
+modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko=
+modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA=
+modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4=
+modernc.org/ccgo/v3 v3.14.0/go.mod h1:hBrkiBlUwvr5vV/ZH9YzXIp982jKE8Ek8tR1ytoAL6Q=
+modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
+modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
+modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8=
+modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I=
+modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
+modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
+modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q=
+modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg=
+modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M=
+modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU=
+modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE=
+modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso=
+modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8=
+modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8=
+modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I=
+modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk=
+modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY=
+modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE=
+modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg=
+modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM=
+modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg=
+modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo=
+modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8=
+modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ=
+modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA=
+modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM=
+modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg=
+modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE=
+modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM=
+modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU=
+modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
+modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
+modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
+modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
+modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
+modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
+modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
+modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE=
+modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY=
+modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ=
+modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
+modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
+modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
+modernc.org/libc v1.13.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
+modernc.org/libc v1.13.2/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
+modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
+modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34=
+modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ=
+modernc.org/libc v1.14.5 h1:DAHvwGoVRDZs5iJXnX9RJrgXSsorupCWmJ2ac964Owk=
+modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak=
+modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
+modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
+modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
+modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
+modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sqlite v1.14.5 h1:bYrrjwH9Y7QUGk1MbchZDhRfmpGuEAs/D45sVjNbfvs=
+modernc.org/sqlite v1.14.5/go.mod h1:YyX5Rx0WbXokitdWl2GJIDy4BrPxBP0PwwhpXOHCDLE=
+modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
+modernc.org/tcl v1.10.0/go.mod h1:WzWapmP/7dHVhFoyPpEaNSVTL8xtewhouN/cqSJ5A2s=
+modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.2.21/go.mod h1:uXrObx4pGqXWIMliC5MiKuwAyMrltzwpteOFUP1PWCc=
+modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g=
 nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
@@ -1705,3 +1842,5 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+zombiezen.com/go/sqlite v0.9.2 h1:bgY6e0BksSrrQqM9gK+HYLbMj1zxcRWHijJBlYnRpCw=
+zombiezen.com/go/sqlite v0.9.2/go.mod h1:M/gb7zbJfWDAUQAsw/9wf0c3P1cHt6Mv9zrtVmAs13Y=
diff --git a/netquery/orm/decoder.go b/netquery/orm/decoder.go
new file mode 100644
index 00000000..5e0d1f7b
--- /dev/null
+++ b/netquery/orm/decoder.go
@@ -0,0 +1,404 @@
+package orm
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+	"time"
+
+	"zombiezen.com/go/sqlite"
+)
+
+// Commonly used error messages when working with orm.
+var (
+	errStructExpected        = errors.New("encode: can only encode structs to maps")
+	errStructPointerExpected = errors.New("decode: result must be pointer to a struct type or map[string]interface{}")
+	errUnexpectedColumnType  = errors.New("decode: unexpected column type")
+)
+
+// constants used when transforming data to and from sqlite.
+var (
+	// sqliteTimeFromat defines the string representation that is
+	// expected by SQLite DATETIME functions.
+	// Note that SQLite itself does not include support for a DATETIME
+	// column type. Instead, dates and times are stored either as INTEGER,
+	// TEXT or REAL.
+	// This package provides support for time.Time being stored as TEXT (using a
+	// preconfigured timezone; UTC by default) or as INTEGER (the user can choose between
+	// unixepoch and unixnano-epoch where the nano variant is not offically supported by
+	// SQLITE).
+	sqliteTimeFormat = "2006-01-02 15:04:05"
+)
+
+type (
+
+	// Stmt describes the interface that must be implemented in order to
+	// be decodable to a struct type using DecodeStmt. This interface is implemented
+	// by *sqlite.Stmt.
+	Stmt interface {
+		ColumnCount() int
+		ColumnName(int) string
+		ColumnType(int) sqlite.ColumnType
+		ColumnText(int) string
+		ColumnBool(int) bool
+		ColumnFloat(int) float64
+		ColumnInt(int) int
+		ColumnReader(int) *bytes.Reader
+	}
+
+	// DecodeFunc is called for each non-basic type during decoding.
+	DecodeFunc func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error)
+
+	DecodeConfig struct {
+		DecodeHooks []DecodeFunc
+	}
+)
+
+// DecodeStmt decodes the current result row loaded in Stmt into the struct or map type result.
+// Decoding hooks configured in cfg are executed before trying to decode basic types and may
+// be specified to provide support for special types.
+// See DatetimeDecoder() for an example of a DecodeHook that handles graceful time.Time conversion.
+//
+func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeConfig) error {
+	// make sure we got something to decode into ...
+	if result == nil {
+		return fmt.Errorf("%w, got %T", errStructPointerExpected, result)
+	}
+
+	// fast path for decoding into a map
+	if mp, ok := result.(*map[string]interface{}); ok {
+		return decodeIntoMap(ctx, stmt, mp)
+	}
+
+	// make sure we got a pointer in result
+	if reflect.TypeOf(result).Kind() != reflect.Ptr {
+		return fmt.Errorf("%w, got %T", errStructPointerExpected, result)
+	}
+
+	// make sure it's a poiter to a struct type
+	t := reflect.ValueOf(result).Elem().Type()
+	if t.Kind() != reflect.Struct {
+		return fmt.Errorf("%w, got %T", errStructPointerExpected, result)
+	}
+
+	// if result is a nil pointer make sure to allocate some space
+	// for the resulting struct
+	resultValue := reflect.ValueOf(result)
+	if resultValue.IsNil() {
+		resultValue.Set(
+			reflect.New(t),
+		)
+	}
+
+	// we need access to the struct directly and not to the
+	// pointer.
+	target := reflect.Indirect(resultValue)
+
+	// create a lookup map from field name (or sqlite:"" tag)
+	// to the field name
+	lm := make(map[string]string)
+	for i := 0; i < target.NumField(); i++ {
+		fieldType := t.Field(i)
+
+		// skip unexported fields
+		if !fieldType.IsExported() {
+			continue
+		}
+
+		lm[sqlColumnName(fieldType)] = fieldType.Name
+	}
+
+	// iterate over all columns and assign them to the correct
+	// fields
+	for i := 0; i < stmt.ColumnCount(); i++ {
+		colName := stmt.ColumnName(i)
+		fieldName, ok := lm[colName]
+		if !ok {
+			// there's no target field for this column
+			// so we can skip it
+			continue
+		}
+		fieldType, _ := t.FieldByName(fieldName)
+
+		value := target.FieldByName(fieldName)
+
+		colType := stmt.ColumnType(i)
+
+		// if the column is reported as NULL we keep
+		// the field as it is.
+		// TODO(ppacher): should we set it to nil here?
+		if colType == sqlite.TypeNull {
+			continue
+		}
+
+		// if value is a nil pointer we need to allocate some memory
+		// first
+		if getKind(value) == reflect.Ptr && value.IsNil() {
+			storage := reflect.New(fieldType.Type.Elem())
+
+			value.Set(storage)
+
+			// make sure value actually points the
+			// dereferenced target storage
+			value = storage.Elem()
+		}
+
+		// execute all decode hooks but make sure we use decodeBasic() as the
+		// last one.
+		columnValue, err := runDecodeHooks(
+			i,
+			stmt,
+			fieldType,
+			value,
+			append(cfg.DecodeHooks, decodeBasic()),
+		)
+		if err != nil {
+			return err
+		}
+
+		// if we don't have a converted value now we try to
+		// decode basic types
+		if columnValue == nil {
+			return fmt.Errorf("cannot decode column %d (type=%s)", i, colType)
+		}
+
+		//log.Printf("valueTypeName: %s fieldName = %s value-orig = %s value = %s (%v) newValue = %s", value.Type().String(), fieldName, target.FieldByName(fieldName).Type(), value.Type(), value, columnValue)
+
+		// convert it to the target type if conversion is possible
+		newValue := reflect.ValueOf(columnValue)
+		if newValue.Type().ConvertibleTo(value.Type()) {
+			newValue = newValue.Convert(value.Type())
+		}
+
+		// assign the new value to the struct field.
+		value.Set(newValue)
+	}
+
+	return nil
+}
+
+// DatetimeDecoder is capable of decoding sqlite INTEGER or TEXT storage classes into
+// time.Time. For INTEGER storage classes, it supports 'unixnano' struct tag value to
+// decide between Unix or UnixNano epoch timestamps.
+//
+// FIXME(ppacher): update comment about loc parameter and TEXT storage class parsing
+//
+func DatetimeDecoder(loc *time.Location) DecodeFunc {
+	return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) {
+		// we only care about "time.Time" here
+		if outval.Type().String() != "time.Time" {
+			return nil, nil
+		}
+
+		switch stmt.ColumnType(colIdx) {
+		case sqlite.TypeInteger:
+			// stored as unix-epoch, if unixnano is set in the struct field tag
+			// we parse it with nano-second resolution
+			// TODO(ppacher): actually split the tag value at "," and search
+			// the slice for "unixnano"
+			if strings.Contains(fieldDef.Tag.Get("sqlite"), ",unixnano") {
+				return time.Unix(0, int64(stmt.ColumnInt(colIdx))), nil
+			}
+
+			return time.Unix(int64(stmt.ColumnInt(colIdx)), 0), nil
+
+		case sqlite.TypeText:
+			// stored ISO8601 but does not have any timezone information
+			// assigned so we always treat it as loc here.
+			t, err := time.ParseInLocation(sqliteTimeFormat, stmt.ColumnText(colIdx), loc)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse %q in %s: %w", stmt.ColumnText(colIdx), fieldDef.Name, err)
+			}
+
+			return t, nil
+
+		case sqlite.TypeFloat:
+			// stored as Julian day numbers
+			return nil, fmt.Errorf("REAL storage type not support for time.Time")
+
+		default:
+			return nil, fmt.Errorf("unsupported storage type for time.Time: %s", outval.Type())
+		}
+	}
+}
+
+func decodeIntoMap(ctx context.Context, stmt Stmt, mp *map[string]interface{}) error {
+	if *mp == nil {
+		*mp = make(map[string]interface{})
+	}
+
+	for i := 0; i < stmt.ColumnCount(); i++ {
+		var x interface{}
+		val, err := decodeBasic()(i, stmt, reflect.StructField{}, reflect.ValueOf(&x).Elem())
+		if err != nil {
+			return fmt.Errorf("failed to decode column %s: %w", stmt.ColumnName(i), err)
+		}
+
+		(*mp)[stmt.ColumnName(i)] = val
+	}
+
+	return nil
+}
+
+func decodeBasic() DecodeFunc {
+	return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) {
+		valueKind := getKind(outval)
+		colType := stmt.ColumnType(colIdx)
+		colName := stmt.ColumnName(colIdx)
+
+		errInvalidType := fmt.Errorf("%w %s for column %s with field type %s", errUnexpectedColumnType, colType.String(), colName, outval.Type())
+
+		switch valueKind {
+		case reflect.String:
+			if colType != sqlite.TypeText {
+				return nil, errInvalidType
+			}
+			return stmt.ColumnText(colIdx), nil
+
+		case reflect.Bool:
+			// sqlite does not have a BOOL type, it rather stores a 1/0 in a column
+			// with INTEGER affinity.
+			if colType != sqlite.TypeInteger {
+				return nil, errInvalidType
+			}
+			return stmt.ColumnBool(colIdx), nil
+
+		case reflect.Float64:
+			if colType != sqlite.TypeFloat {
+				return nil, errInvalidType
+			}
+			return stmt.ColumnFloat(colIdx), nil
+
+		case reflect.Int, reflect.Uint: // getKind() normalizes all ints to reflect.Int/Uint because sqlite doesn't really care ...
+			if colType != sqlite.TypeInteger {
+				return nil, errInvalidType
+			}
+
+			return stmt.ColumnInt(colIdx), nil
+
+		case reflect.Slice:
+			if outval.Type().Elem().Kind() != reflect.Uint8 {
+				return nil, fmt.Errorf("slices other than []byte for BLOB are not supported")
+			}
+
+			if colType != sqlite.TypeBlob {
+				return nil, errInvalidType
+			}
+
+			columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx))
+			if err != nil {
+				return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
+			}
+
+			return columnValue, nil
+
+		case reflect.Interface:
+			var (
+				t reflect.Type
+				x interface{}
+			)
+			switch colType {
+			case sqlite.TypeBlob:
+				t = reflect.TypeOf([]byte{})
+				columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx))
+				if err != nil {
+					return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
+				}
+				x = columnValue
+
+			case sqlite.TypeFloat:
+				t = reflect.TypeOf(float64(0))
+				x = stmt.ColumnFloat(colIdx)
+
+			case sqlite.TypeInteger:
+				t = reflect.TypeOf(int(0))
+				x = stmt.ColumnInt(colIdx)
+
+			case sqlite.TypeText:
+				t = reflect.TypeOf(string(""))
+				x = stmt.ColumnText(colIdx)
+
+			case sqlite.TypeNull:
+				t = nil
+				x = nil
+
+			default:
+				return nil, fmt.Errorf("unsupported column type %s", colType)
+			}
+
+			if t == nil {
+				return nil, nil
+			}
+
+			target := reflect.New(t).Elem()
+			target.Set(reflect.ValueOf(x))
+
+			return target.Interface(), nil
+
+		default:
+			return nil, fmt.Errorf("cannot decode into %s", valueKind)
+		}
+	}
+}
+
+func sqlColumnName(fieldType reflect.StructField) string {
+	tagValue, hasTag := fieldType.Tag.Lookup("sqlite")
+	if !hasTag {
+		return fieldType.Name
+	}
+
+	parts := strings.Split(tagValue, ",")
+	if parts[0] != "" {
+		return parts[0]
+	}
+
+	return fieldType.Name
+}
+
+// runDecodeHooks tries to decode the column value of stmt at index colIdx into outval by running all decode hooks.
+// The first hook that returns a non-nil interface wins, other hooks will not be executed. If an error is
+// returned by a decode hook runDecodeHooks stops the error is returned to the caller.
+func runDecodeHooks(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value, hooks []DecodeFunc) (interface{}, error) {
+	for _, fn := range hooks {
+		res, err := fn(colIdx, stmt, fieldDef, outval)
+		if err != nil {
+			return res, err
+		}
+
+		if res != nil {
+			return res, nil
+		}
+	}
+
+	return nil, nil
+}
+
+// getKind returns the kind of value but normalized Int, Uint and Float varaints
+// to their base type.
+func getKind(val reflect.Value) reflect.Kind {
+	kind := val.Kind()
+	return normalizeKind(kind)
+}
+
+func normalizeKind(kind reflect.Kind) reflect.Kind {
+	switch {
+	case kind >= reflect.Int && kind <= reflect.Int64:
+		return reflect.Int
+	case kind >= reflect.Uint && kind <= reflect.Uint64:
+		return reflect.Uint
+	case kind >= reflect.Float32 && kind <= reflect.Float64:
+		return reflect.Float64
+	default:
+		return kind
+	}
+}
+
+var DefaultDecodeConfig = DecodeConfig{
+	DecodeHooks: []DecodeFunc{
+		DatetimeDecoder(time.UTC),
+	},
+}
diff --git a/netquery/orm/decoder_test.go b/netquery/orm/decoder_test.go
new file mode 100644
index 00000000..c6c836c0
--- /dev/null
+++ b/netquery/orm/decoder_test.go
@@ -0,0 +1,475 @@
+package orm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"zombiezen.com/go/sqlite"
+)
+
+type testStmt struct {
+	columns []string
+	values  []interface{}
+	types   []sqlite.ColumnType
+}
+
+func (ts testStmt) ColumnCount() int                   { return len(ts.columns) }
+func (ts testStmt) ColumnName(i int) string            { return ts.columns[i] }
+func (ts testStmt) ColumnBool(i int) bool              { return ts.values[i].(bool) }
+func (ts testStmt) ColumnText(i int) string            { return ts.values[i].(string) }
+func (ts testStmt) ColumnFloat(i int) float64          { return ts.values[i].(float64) }
+func (ts testStmt) ColumnInt(i int) int                { return ts.values[i].(int) }
+func (ts testStmt) ColumnReader(i int) *bytes.Reader   { return bytes.NewReader(ts.values[i].([]byte)) }
+func (ts testStmt) ColumnType(i int) sqlite.ColumnType { return ts.types[i] }
+
+// compile time check
+var _ Stmt = new(testStmt)
+
+type exampleFieldTypes struct {
+	S string
+	I int
+	F float64
+	B bool
+}
+
+type examplePointerTypes struct {
+	S *string
+	I *int
+	F *float64
+	B *bool
+}
+
+type exampleStructTags struct {
+	S string `sqlite:"col_string"`
+	I int    `sqlite:"col_int"`
+}
+
+type exampleIntConv struct {
+	I8  int8
+	I16 int16
+	I32 int32
+	I64 int64
+	I   int
+}
+
+type exampleBlobTypes struct {
+	B []byte
+}
+
+type exampleJSONRawTypes struct {
+	B json.RawMessage
+}
+
+type exampleTimeTypes struct {
+	T  time.Time
+	TP *time.Time
+}
+
+type exampleInterface struct {
+	I  interface{}
+	IP *interface{}
+}
+
+func (ett *exampleTimeTypes) Equal(other interface{}) bool {
+	oett, ok := other.(*exampleTimeTypes)
+	if !ok {
+		return false
+	}
+	return ett.T.Equal(oett.T) && (ett.TP != nil && oett.TP != nil && ett.TP.Equal(*oett.TP)) || (ett.TP == nil && oett.TP == nil)
+}
+
+type exampleTimeNano struct {
+	T time.Time `sqlite:",unixnano"`
+}
+
+func (etn *exampleTimeNano) Equal(other interface{}) bool {
+	oetn, ok := other.(*exampleTimeNano)
+	if !ok {
+		return false
+	}
+	return etn.T.Equal(oetn.T)
+}
+
+func Test_Decoder(t *testing.T) {
+	ctx := context.TODO()
+
+	refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC)
+
+	cases := []struct {
+		Desc     string
+		Stmt     testStmt
+		Result   interface{}
+		Expected interface{}
+	}{
+		{
+			"Decoding into nil is not allowed",
+			testStmt{
+				columns: nil,
+				values:  nil,
+				types:   nil,
+			},
+			nil,
+			nil,
+		},
+		{
+			"Decoding into basic types",
+			testStmt{
+				columns: []string{"S", "I", "F", "B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeText,
+					sqlite.TypeInteger,
+					sqlite.TypeFloat,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					"string value",
+					1,
+					1.2,
+					true,
+				},
+			},
+			&exampleFieldTypes{},
+			&exampleFieldTypes{
+				S: "string value",
+				I: 1,
+				F: 1.2,
+				B: true,
+			},
+		},
+		{
+			"Decoding into basic types with different order",
+			testStmt{
+				columns: []string{"I", "S", "B", "F"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeText,
+					sqlite.TypeInteger,
+					sqlite.TypeFloat,
+				},
+				values: []interface{}{
+					1,
+					"string value",
+					true,
+					1.2,
+				},
+			},
+			&exampleFieldTypes{},
+			&exampleFieldTypes{
+				S: "string value",
+				I: 1,
+				F: 1.2,
+				B: true,
+			},
+		},
+		{
+			"Decoding into basic types with missing values",
+			testStmt{
+				columns: []string{"F", "B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeFloat,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					1.2,
+					true,
+				},
+			},
+			&exampleFieldTypes{},
+			&exampleFieldTypes{
+				F: 1.2,
+				B: true,
+			},
+		},
+		{
+			"Decoding into pointer types",
+			testStmt{
+				columns: []string{"S", "I", "F", "B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeText,
+					sqlite.TypeInteger,
+					sqlite.TypeFloat,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					"string value",
+					1,
+					1.2,
+					true,
+				},
+			},
+			&examplePointerTypes{},
+			func() interface{} {
+				s := "string value"
+				i := 1
+				f := 1.2
+				b := true
+
+				return &examplePointerTypes{
+					S: &s,
+					I: &i,
+					F: &f,
+					B: &b,
+				}
+			},
+		},
+		{
+			"Decoding into pointer types with missing values",
+			testStmt{
+				columns: []string{"S", "B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeText,
+					sqlite.TypeInteger,
+					sqlite.TypeFloat,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					"string value",
+					true,
+				},
+			},
+			&examplePointerTypes{},
+			func() interface{} {
+				s := "string value"
+				b := true
+
+				return &examplePointerTypes{
+					S: &s,
+					B: &b,
+				}
+			},
+		},
+		{
+			"Decoding into fields with struct tags",
+			testStmt{
+				columns: []string{"col_string", "col_int"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeText,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					"string value",
+					1,
+				},
+			},
+			&exampleStructTags{},
+			&exampleStructTags{
+				S: "string value",
+				I: 1,
+			},
+		},
+		{
+			"Decoding into correct int type",
+			testStmt{
+				columns: []string{"I8", "I16", "I32", "I64", "I"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					1,
+					1,
+					1,
+					1,
+					1,
+				},
+			},
+			&exampleIntConv{},
+			&exampleIntConv{
+				1, 1, 1, 1, 1,
+			},
+		},
+		{
+			"Handling NULL values for basic types",
+			testStmt{
+				columns: []string{"S", "I", "F"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeNull,
+					sqlite.TypeNull,
+					sqlite.TypeFloat,
+				},
+				values: []interface{}{
+					// we use nil here but actually that does not matter
+					nil,
+					nil,
+					1.0,
+				},
+			},
+			&exampleFieldTypes{},
+			&exampleFieldTypes{
+				F: 1.0,
+			},
+		},
+		{
+			"Handling NULL values for pointer types",
+			testStmt{
+				columns: []string{"S", "I", "F"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeNull,
+					sqlite.TypeNull,
+					sqlite.TypeFloat,
+				},
+				values: []interface{}{
+					// we use nil here but actually that does not matter
+					nil,
+					nil,
+					1.0,
+				},
+			},
+			&examplePointerTypes{},
+			func() interface{} {
+				f := 1.0
+
+				return &examplePointerTypes{F: &f}
+			},
+		},
+		{
+			"Handling blob types",
+			testStmt{
+				columns: []string{"B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeBlob,
+				},
+				values: []interface{}{
+					([]byte)("hello world"),
+				},
+			},
+			&exampleBlobTypes{},
+			&exampleBlobTypes{
+				B: ([]byte)("hello world"),
+			},
+		},
+		{
+			"Handling blob types as json.RawMessage",
+			testStmt{
+				columns: []string{"B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeBlob,
+				},
+				values: []interface{}{
+					([]byte)("hello world"),
+				},
+			},
+			&exampleJSONRawTypes{},
+			&exampleJSONRawTypes{
+				B: (json.RawMessage)("hello world"),
+			},
+		},
+		{
+			"Handling time.Time and pointers to it",
+			testStmt{
+				columns: []string{"T", "TP"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					int(refTime.Unix()),
+					int(refTime.Unix()),
+				},
+			},
+			&exampleTimeTypes{},
+			&exampleTimeTypes{
+				T:  refTime,
+				TP: &refTime,
+			},
+		},
+		{
+			"Handling time.Time in nano-second resolution (struct tags)",
+			testStmt{
+				columns: []string{"T", "TP"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeInteger,
+				},
+				values: []interface{}{
+					int(refTime.UnixNano()),
+					int(refTime.UnixNano()),
+				},
+			},
+			&exampleTimeNano{},
+			&exampleTimeNano{
+				T: refTime,
+			},
+		},
+		{
+			"Decoding into interface",
+			testStmt{
+				columns: []string{"I", "IP"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeText,
+					sqlite.TypeText,
+				},
+				values: []interface{}{
+					"value1",
+					"value2",
+				},
+			},
+			&exampleInterface{},
+			func() interface{} {
+				var x interface{}
+				x = "value2"
+
+				return &exampleInterface{
+					I:  "value1",
+					IP: &x,
+				}
+			},
+		},
+		{
+			"Decoding into map[string]interface{}",
+			testStmt{
+				columns: []string{"I", "F", "S", "B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeFloat,
+					sqlite.TypeText,
+					sqlite.TypeBlob,
+				},
+				values: []interface{}{
+					1,
+					1.1,
+					"string value",
+					[]byte("blob value"),
+				},
+			},
+			new(map[string]interface{}),
+			&map[string]interface{}{
+				"I": 1,
+				"F": 1.1,
+				"S": "string value",
+				"B": []byte("blob value"),
+			},
+		},
+	}
+
+	for idx := range cases {
+		c := cases[idx]
+		t.Run(c.Desc, func(t *testing.T) {
+			t.Parallel()
+
+			err := DecodeStmt(ctx, c.Stmt, c.Result, DefaultDecodeConfig)
+			if fn, ok := c.Expected.(func() interface{}); ok {
+				c.Expected = fn()
+			}
+
+			if c.Expected == nil {
+				assert.Error(t, err, c.Desc)
+			} else {
+				assert.NoError(t, err, c.Desc)
+
+				if equaler, ok := c.Expected.(interface{ Equal(x interface{}) bool }); ok {
+					assert.True(t, equaler.Equal(c.Result))
+				} else {
+					assert.Equal(t, c.Expected, c.Result)
+				}
+			}
+		})
+	}
+}
diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go
new file mode 100644
index 00000000..c2d9a62c
--- /dev/null
+++ b/netquery/orm/encoder.go
@@ -0,0 +1,124 @@
+package orm
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"time"
+
+	"zombiezen.com/go/sqlite"
+)
+
+type (
+	EncodeFunc func(col *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error)
+
+	EncodeConfig struct {
+		EncodeHooks []EncodeFunc
+	}
+)
+
+// EncodeAsMap returns a map that contains the value of each struct field of
+// r using the sqlite column name as a map key. It either uses the name of the
+// exported struct field or the value of the "sqlite" tag.
+func EncodeAsMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig) (map[string]interface{}, error) {
+	// make sure we work on a struct type
+	val := reflect.Indirect(reflect.ValueOf(r))
+	if val.Kind() != reflect.Struct {
+		return nil, fmt.Errorf("%w, got %T", errStructExpected, r)
+	}
+
+	res := make(map[string]interface{}, val.NumField())
+
+	for i := 0; i < val.NumField(); i++ {
+		fieldType := val.Type().Field(i)
+		field := val.Field(i)
+
+		// skip unexported fields
+		if !fieldType.IsExported() {
+			continue
+		}
+
+		colDev, err := getColumnDef(fieldType)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get column definition for %s: %w", fieldType.Name, err)
+		}
+
+		x, found, err := runEncodeHooks(colDev, fieldType.Type, field, cfg.EncodeHooks)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run encode hooks: %w", err)
+		}
+
+		if !found {
+			if reflect.Indirect(field).IsValid() {
+				x = reflect.Indirect(field).Interface()
+			}
+		}
+
+		res[keyPrefix+sqlColumnName(fieldType)] = x
+
+	}
+
+	return res, nil
+}
+
+func DatetimeEncoder(loc *time.Location) EncodeFunc {
+	return func(colDev *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) {
+		// if fieldType holds a pointer we need to dereference the value
+		ft := valType.String()
+		if valType.Kind() == reflect.Ptr {
+			ft = valType.Elem().String()
+			val = reflect.Indirect(val)
+		}
+
+		// we only care about "time.Time" here
+		if ft != "time.Time" {
+			return nil, false, nil
+		}
+
+		// handle the zero time as a NULL.
+		if !val.IsValid() || val.IsZero() {
+			return nil, true, nil
+		}
+
+		valInterface := val.Interface()
+		t, ok := valInterface.(time.Time)
+		if !ok {
+			return nil, false, fmt.Errorf("cannot convert reflect value to time.Time")
+		}
+
+		switch colDev.Type {
+		case sqlite.TypeInteger:
+			if colDev.UnixNano {
+				return t.UnixNano(), true, nil
+			}
+			return t.Unix(), true, nil
+		case sqlite.TypeText:
+			str := t.In(loc).Format(sqliteTimeFormat)
+
+			return str, true, nil
+		}
+
+		return nil, false, fmt.Errorf("cannot store time.Time in %s", colDev.Type)
+	}
+}
+
+func runEncodeHooks(colDev *ColumnDef, valType reflect.Type, val reflect.Value, hooks []EncodeFunc) (interface{}, bool, error) {
+	for _, fn := range hooks {
+		res, end, err := fn(colDev, valType, val)
+		if err != nil {
+			return res, false, err
+		}
+
+		if end {
+			return res, true, nil
+		}
+	}
+
+	return nil, false, nil
+}
+
+var DefaultEncodeConfig = EncodeConfig{
+	EncodeHooks: []EncodeFunc{
+		DatetimeEncoder(time.UTC),
+	},
+}
diff --git a/netquery/orm/encoder_test.go b/netquery/orm/encoder_test.go
new file mode 100644
index 00000000..056bf953
--- /dev/null
+++ b/netquery/orm/encoder_test.go
@@ -0,0 +1,126 @@
+package orm
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_EncodeAsMap(t *testing.T) {
+	ctx := context.TODO()
+	refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC)
+
+	cases := []struct {
+		Desc     string
+		Input    interface{}
+		Expected map[string]interface{}
+	}{
+		{
+			"Encode basic types",
+			struct {
+				I int
+				F float64
+				S string
+				B []byte
+			}{
+				I: 1,
+				F: 1.2,
+				S: "string",
+				B: ([]byte)("bytes"),
+			},
+			map[string]interface{}{
+				"I": 1,
+				"F": 1.2,
+				"S": "string",
+				"B": ([]byte)("bytes"),
+			},
+		},
+		{
+			"Encode using struct tags",
+			struct {
+				I int    `sqlite:"col_int"`
+				S string `sqlite:"col_string"`
+			}{
+				I: 1,
+				S: "string value",
+			},
+			map[string]interface{}{
+				"col_int":    1,
+				"col_string": "string value",
+			},
+		},
+		{
+			"Ignore Private fields",
+			struct {
+				I int
+				s string
+			}{
+				I: 1,
+				s: "string value",
+			},
+			map[string]interface{}{
+				"I": 1,
+			},
+		},
+		{
+			"Handle Pointers",
+			struct {
+				I *int
+				S *string
+			}{
+				I: new(int),
+			},
+			map[string]interface{}{
+				"I": 0,
+				"S": nil,
+			},
+		},
+		{
+			"Handle time.Time types",
+			struct {
+				TinInt    time.Time `sqlite:",integer,unixnano"`
+				TinString time.Time `sqlite:",text"`
+			}{
+				TinInt:    refTime,
+				TinString: refTime,
+			},
+			map[string]interface{}{
+				"TinInt":    refTime.UnixNano(),
+				"TinString": refTime.Format(sqliteTimeFormat),
+			},
+		},
+		{
+			"Handle time.Time pointer types",
+			struct {
+				TinInt    *time.Time `sqlite:",integer,unixnano"`
+				TinString *time.Time `sqlite:",text"`
+				Tnil1     *time.Time `sqlite:",text"`
+				Tnil2     *time.Time `sqlite:",text"`
+			}{
+				TinInt:    &refTime,
+				TinString: &refTime,
+				Tnil1:     nil,
+				Tnil2:     (*time.Time)(nil),
+			},
+			map[string]interface{}{
+				"TinInt":    refTime.UnixNano(),
+				"TinString": refTime.Format(sqliteTimeFormat),
+				"Tnil1":     nil,
+				"Tnil2":     nil,
+			},
+		},
+	}
+
+	for idx := range cases {
+		c := cases[idx]
+		t.Run(c.Desc, func(t *testing.T) {
+			// t.Parallel()
+
+			res, err := EncodeAsMap(ctx, c.Input, "", DefaultEncodeConfig)
+			assert.NoError(t, err)
+			assert.Equal(t, c.Expected, res)
+		})
+	}
+}
diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go
new file mode 100644
index 00000000..d94e62db
--- /dev/null
+++ b/netquery/orm/query_runner.go
@@ -0,0 +1,123 @@
+package orm
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"zombiezen.com/go/sqlite"
+	"zombiezen.com/go/sqlite/sqlitex"
+)
+
+type (
+	QueryOption func(opts *queryOpts)
+
+	queryOpts struct {
+		Transient    bool
+		Args         []interface{}
+		NamedArgs    map[string]interface{}
+		Result       interface{}
+		DecodeConfig DecodeConfig
+	}
+)
+
+func WithTransient() QueryOption {
+	return func(opts *queryOpts) {
+		opts.Transient = true
+	}
+}
+
+func WithArgs(args ...interface{}) QueryOption {
+	return func(opts *queryOpts) {
+		opts.Args = args
+	}
+}
+
+func WithNamedArgs(args map[string]interface{}) QueryOption {
+	return func(opts *queryOpts) {
+		opts.NamedArgs = args
+	}
+}
+
+func WithResult(result interface{}) QueryOption {
+	return func(opts *queryOpts) {
+		opts.Result = result
+	}
+}
+
+func WithDecodeConfig(cfg DecodeConfig) QueryOption {
+	return func(opts *queryOpts) {
+		opts.DecodeConfig = cfg
+	}
+}
+
+func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...QueryOption) error {
+	args := queryOpts{
+		DecodeConfig: DefaultDecodeConfig,
+	}
+
+	for _, fn := range modifiers {
+		fn(&args)
+	}
+
+	opts := &sqlitex.ExecOptions{
+		Args:  args.Args,
+		Named: args.NamedArgs,
+	}
+
+	var (
+		sliceVal    reflect.Value
+		valElemType reflect.Type
+	)
+
+	if args.Result != nil {
+		target := args.Result
+		outVal := reflect.ValueOf(target)
+		if outVal.Kind() != reflect.Ptr {
+			return fmt.Errorf("target must be a pointer, got %T", target)
+		}
+
+		sliceVal = reflect.Indirect(outVal)
+		if !sliceVal.IsValid() || sliceVal.IsNil() {
+			newVal := reflect.Zero(outVal.Type().Elem())
+			sliceVal.Set(newVal)
+		}
+
+		kind := sliceVal.Kind()
+		if kind != reflect.Slice {
+			return fmt.Errorf("target but be pointer to slice, got %T", target)
+		}
+		valType := sliceVal.Type()
+		valElemType = valType.Elem()
+
+		opts.ResultFunc = func(stmt *sqlite.Stmt) error {
+			var currentField reflect.Value
+
+			currentField = reflect.New(valElemType)
+
+			if err := DecodeStmt(ctx, stmt, currentField.Interface(), args.DecodeConfig); err != nil {
+				return err
+			}
+
+			sliceVal = reflect.Append(sliceVal, reflect.Indirect(currentField))
+
+			return nil
+		}
+	}
+
+	var err error
+	if args.Transient {
+		err = sqlitex.ExecuteTransient(conn, sql, opts)
+	} else {
+		err = sqlitex.Execute(conn, sql, opts)
+	}
+	if err != nil {
+		return err
+	}
+
+	if args.Result != nil {
+		reflect.Indirect(reflect.ValueOf(args.Result)).Set(sliceVal)
+	}
+
+	return nil
+}
diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go
new file mode 100644
index 00000000..aa57e21a
--- /dev/null
+++ b/netquery/orm/schema_builder.go
@@ -0,0 +1,220 @@
+package orm
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"zombiezen.com/go/sqlite"
+)
+
+var (
+	errSkipStructField = errors.New("struct field should be skipped")
+)
+
+var (
+	TagUnixNano          = "unixnano"
+	TagPrimaryKey        = "primary"
+	TagAutoIncrement     = "autoincrement"
+	TagNotNull           = "not-null"
+	TagNullable          = "nullable"
+	TagTypeInt           = "integer"
+	TagTypeText          = "text"
+	TagTypePrefixVarchar = "varchar"
+	TagTypeBlob          = "blob"
+	TagTypeFloat         = "float"
+)
+
+var sqlTypeMap = map[sqlite.ColumnType]string{
+	sqlite.TypeBlob:    "BLOB",
+	sqlite.TypeFloat:   "REAL",
+	sqlite.TypeInteger: "INTEGER",
+	sqlite.TypeText:    "TEXT",
+}
+
+type (
+	TableSchema struct {
+		Name    string
+		Columns []ColumnDef
+	}
+
+	ColumnDef struct {
+		Name          string
+		Nullable      bool
+		Type          sqlite.ColumnType
+		Length        int
+		PrimaryKey    bool
+		AutoIncrement bool
+		UnixNano      bool
+	}
+)
+
+func (ts TableSchema) CreateStatement(ifNotExists bool) string {
+	sql := "CREATE TABLE"
+	if ifNotExists {
+		sql += " IF NOT EXISTS"
+	}
+	sql += " " + ts.Name + " ( "
+
+	for idx, col := range ts.Columns {
+		sql += col.AsSQL()
+		if idx < len(ts.Columns)-1 {
+			sql += ", "
+		}
+	}
+
+	sql += " );"
+	return sql
+}
+
+func (def ColumnDef) AsSQL() string {
+	sql := def.Name + " "
+
+	if def.Type == sqlite.TypeText && def.Length > 0 {
+		sql += fmt.Sprintf("VARCHAR(%d)", def.Length)
+	} else {
+		sql += sqlTypeMap[def.Type]
+	}
+
+	if def.PrimaryKey {
+		sql += " PRIMARY KEY"
+	}
+	if def.AutoIncrement {
+		sql += " AUTOINCREMENT"
+	}
+	if !def.Nullable {
+		sql += " NOT NULL"
+	}
+
+	return sql
+}
+
+func GenerateTableSchema(name string, d interface{}) (*TableSchema, error) {
+	ts := &TableSchema{
+		Name: name,
+	}
+
+	val := reflect.Indirect(reflect.ValueOf(d))
+	if val.Kind() != reflect.Struct {
+		return nil, fmt.Errorf("%w, got %T", errStructExpected, d)
+	}
+
+	for i := 0; i < val.NumField(); i++ {
+		fieldType := val.Type().Field(i)
+		if !fieldType.IsExported() {
+			continue
+		}
+
+		def, err := getColumnDef(fieldType)
+		if err != nil {
+			if errors.Is(err, errSkipStructField) {
+				continue
+			}
+
+			return nil, fmt.Errorf("struct field %s: %w", fieldType.Name, err)
+		}
+
+		ts.Columns = append(ts.Columns, *def)
+	}
+
+	return ts, nil
+}
+
+func getColumnDef(fieldType reflect.StructField) (*ColumnDef, error) {
+	def := &ColumnDef{
+		Name:     fieldType.Name,
+		Nullable: fieldType.Type.Kind() == reflect.Ptr,
+	}
+
+	ft := fieldType.Type
+
+	if fieldType.Type.Kind() == reflect.Ptr {
+		ft = fieldType.Type.Elem()
+	}
+
+	kind := normalizeKind(ft.Kind())
+
+	switch kind {
+	case reflect.Int:
+		def.Type = sqlite.TypeInteger
+
+	case reflect.Float64:
+		def.Type = sqlite.TypeFloat
+
+	case reflect.String:
+		def.Type = sqlite.TypeText
+
+	case reflect.Slice:
+		// only []byte/[]uint8 is supported
+		if ft.Elem().Kind() != reflect.Uint8 {
+			return nil, fmt.Errorf("slices of type %s is not supported", ft.Elem())
+		}
+
+		def.Type = sqlite.TypeBlob
+	}
+
+	if err := applyStructFieldTag(fieldType, def); err != nil {
+		return nil, err
+	}
+
+	return def, nil
+}
+
+// applyStructFieldTag parses the sqlite:"" struct field tag and update the column
+// definition def accordingly.
+func applyStructFieldTag(fieldType reflect.StructField, def *ColumnDef) error {
+	parts := strings.Split(fieldType.Tag.Get("sqlite"), ",")
+	if len(parts) > 0 && parts[0] != "" {
+		if parts[0] == "-" {
+			return errSkipStructField
+		}
+
+		def.Name = parts[0]
+	}
+
+	if len(parts) > 1 {
+		for _, k := range parts[1:] {
+			switch k {
+			// column modifieres
+			case TagPrimaryKey:
+				def.PrimaryKey = true
+			case TagAutoIncrement:
+				def.AutoIncrement = true
+			case TagNotNull:
+				def.Nullable = false
+			case TagNullable:
+				def.Nullable = true
+			case TagUnixNano:
+				def.UnixNano = true
+
+			// basic column types
+			case TagTypeInt:
+				def.Type = sqlite.TypeInteger
+			case TagTypeText:
+				def.Type = sqlite.TypeText
+			case TagTypeFloat:
+				def.Type = sqlite.TypeFloat
+			case TagTypeBlob:
+				def.Type = sqlite.TypeBlob
+
+			// advanced column types
+			default:
+				if strings.HasPrefix(k, TagTypePrefixVarchar) {
+					lenStr := strings.TrimSuffix(strings.TrimPrefix(k, TagTypePrefixVarchar+"("), ")")
+					length, err := strconv.ParseInt(lenStr, 10, 0)
+					if err != nil {
+						return fmt.Errorf("failed to parse varchar length %q: %w", lenStr, err)
+					}
+
+					def.Type = sqlite.TypeText
+					def.Length = int(length)
+				}
+
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/netquery/orm/schema_builder_test.go b/netquery/orm/schema_builder_test.go
new file mode 100644
index 00000000..7012076d
--- /dev/null
+++ b/netquery/orm/schema_builder_test.go
@@ -0,0 +1,41 @@
+package orm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_SchemaBuilder(t *testing.T) {
+	cases := []struct {
+		Name        string
+		Model       interface{}
+		ExpectedSQL string
+	}{
+		{
+			"Simple",
+			struct {
+				ID    int         `sqlite:"id,primary,autoincrement"`
+				Text  string      `sqlite:"text,nullable"`
+				Int   *int        `sqlite:",not-null"`
+				Float interface{} `sqlite:",float,nullable"`
+			}{},
+			`CREATE TABLE Simple ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, text TEXT, Int INTEGER NOT NULL, Float REAL );`,
+		},
+		{
+			"Varchar",
+			struct {
+				S string `sqlite:",varchar(10)"`
+			}{},
+			`CREATE TABLE Varchar ( S VARCHAR(10) NOT NULL );`,
+		},
+	}
+
+	for idx := range cases {
+		c := cases[idx]
+
+		res, err := GenerateTableSchema(c.Name, c.Model)
+		assert.NoError(t, err)
+		assert.Equal(t, c.ExpectedSQL, res.CreateStatement(false))
+	}
+}

From 4e3e17fa8ca0e06d7d6e4fe036902da033e2e9d5 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Wed, 16 Mar 2022 20:38:51 +0100
Subject: [PATCH 53/86] Add initial sqlite support in netquery

---
 netquery/database.go             | 298 +++++++++++++++++++++++++++++++
 netquery/manager.go              | 225 +++++++++++++++++++++++
 netquery/module_api.go           | 140 +++++++++++++++
 netquery/runtime_query_runner.go |  77 ++++++++
 4 files changed, 740 insertions(+)
 create mode 100644 netquery/database.go
 create mode 100644 netquery/manager.go
 create mode 100644 netquery/module_api.go
 create mode 100644 netquery/runtime_query_runner.go

diff --git a/netquery/database.go b/netquery/database.go
new file mode 100644
index 00000000..7485f19c
--- /dev/null
+++ b/netquery/database.go
@@ -0,0 +1,298 @@
+package netquery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/netquery/orm"
+	"github.com/safing/portmaster/network"
+	"github.com/safing/portmaster/network/netutils"
+	"github.com/safing/portmaster/network/packet"
+	"zombiezen.com/go/sqlite"
+	"zombiezen.com/go/sqlite/sqlitex"
+)
+
+// InMemory is the "file path" to open a new in-memory database.
+const InMemory = ":memory:"
+
+// Available connection types as their string representation.
+const (
+	ConnTypeDNS = "dns"
+	ConnTypeIP  = "ip"
+)
+
+// ConnectionTypeToString is a lookup map to get the string representation
+// of a network.ConnectionType as used by this package.
+var ConnectionTypeToString = map[network.ConnectionType]string{
+	network.DNSRequest:   ConnTypeDNS,
+	network.IPConnection: ConnTypeIP,
+}
+
+type (
+	// Database represents a SQLite3 backed connection database.
+	// It's use is tailored for persistance and querying of network.Connection.
+	// Access to the underlying SQLite database is synchronized.
+	//
+	// TODO(ppacher): somehow I'm receiving SIGBUS or SIGSEGV when no doing
+	// synchronization in *Database. Check what exactly sqlite.OpenFullMutex, etc..
+	// are actually supposed to do.
+	//
+	Database struct {
+		l    sync.Mutex
+		conn *sqlite.Conn
+	}
+
+	// Conn is a network connection that is stored in a SQLite database and accepted
+	// by the *Database type of this package. This also defines, using the ./orm package,
+	// the table schema and the model that is exposed via the runtime database as well as
+	// the query API.
+	//
+	// Use ConvertConnection from this package to convert a network.Connection to this
+	// representation.
+	Conn struct {
+		// ID is a device-unique identifier for the connection. It is built
+		// from network.Connection by hashing the connection ID and the start
+		// time. We cannot just use the network.Connection.ID because it is only unique
+		// as long as the connection is still active and might be, although unlikely,
+		// reused afterwards.
+		ID         string            `sqlite:"id,primary"`
+		Type       string            `sqlite:"type,varchar(8)"`
+		External   bool              `sqlite:"external"`
+		IPVersion  packet.IPVersion  `sqlite:"ip_version"`
+		IPProtocol packet.IPProtocol `sqlite:"ip_protocol"`
+		LocalIP    string            `sqlite:"local_ip"`
+		LocalPort  uint16            `sqlite:"local_port"`
+		RemoteIP   string            `sqlite:"remote_ip"`
+		RemotePort uint16            `sqlite:"remote_port"`
+		Domain     string            `sqlite:"domain"`
+		Country    string            `sqlite:"country,varchar(2)"`
+		ASN        uint              `sqlite:"asn"`
+		ASOwner    string            `sqlite:"as_owner"`
+		Latitude   float64           `sqlite:"latitude"`
+		Longitude  float64           `sqlite:"longitude"`
+		Scope      netutils.IPScope  `sqlite:"scope"`
+		Verdict    network.Verdict   `sqlite:"verdict"`
+		Started    time.Time         `sqlite:"started,text"`
+		Ended      *time.Time        `sqlite:"ended,text"`
+		Tunneled   bool              `sqlite:"tunneled"`
+		Encrypted  bool              `sqlite:"encrypted"`
+		Internal   bool              `sqlite:"internal"`
+		Inbound    bool              `sqlite:"inbound"`
+		ExtraData  json.RawMessage   `sqlite:"extra_data"`
+	}
+)
+
+// New opens a new database at path. The database is opened with Full-Mutex, Write-Ahead-Log (WAL)
+// and Shared-Cache enabled.
+//
+// TODO(ppacher): check which sqlite "open flags" provide the best performance and don't cause
+// SIGBUS/SIGSEGV when used with out a dedicated mutex in *Database.
+//
+func New(path string) (*Database, error) {
+	c, err := sqlite.OpenConn(
+		path,
+		sqlite.OpenCreate,
+		sqlite.OpenReadWrite,
+		sqlite.OpenFullMutex,
+		sqlite.OpenWAL,
+		sqlite.OpenSharedCache,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open sqlite at %s: %w", path, err)
+	}
+
+	return &Database{conn: c}, nil
+}
+
+// NewInMemory is like New but creates a new in-memory database and
+// automatically applies the connection table schema.
+func NewInMemory() (*Database, error) {
+	db, err := New(InMemory)
+	if err != nil {
+		return nil, err
+	}
+
+	// this should actually never happen because an in-memory database
+	// always starts empty...
+	if err := db.ApplyMigrations(); err != nil {
+		return nil, fmt.Errorf("failed to prepare database: %w", err)
+	}
+
+	return db, nil
+}
+
+// ApplyMigrations applies any table and data migrations that are needed
+// to bring db up-to-date with the built-in schema.
+// TODO(ppacher): right now this only applies the current schema and ignores
+// any data-migrations. Once the history module is implemented this should
+// become/use a full migration system -- use zombiezen.com/go/sqlite/sqlitemigration
+func (db *Database) ApplyMigrations() error {
+	schema, err := orm.GenerateTableSchema("connections", Conn{})
+	if err != nil {
+		return fmt.Errorf("failed to generate table schema for conncetions: %w", err)
+	}
+
+	// get the create-table SQL statement from the infered schema
+	sql := schema.CreateStatement(false)
+
+	// execute the SQL
+	if err := sqlitex.ExecuteTransient(db.conn, sql, nil); err != nil {
+		return fmt.Errorf("failed to create schema: %w", err)
+	}
+
+	return nil
+}
+
+// Execute executes a custom SQL query against the SQLite database used by db.
+// It uses orm.RunQuery() under the hood so please refer to the orm package for
+// more information about available options.
+func (db *Database) Execute(ctx context.Context, sql string, args ...orm.QueryOption) error {
+	db.l.Lock()
+	defer db.l.Unlock()
+
+	return orm.RunQuery(ctx, db.conn, sql, args...)
+}
+
+// CountRows returns the number of rows stored in the database.
+func (db *Database) CountRows(ctx context.Context) (int, error) {
+	var result []struct {
+		Count int `sqlite:"count"`
+	}
+
+	if err := db.Execute(ctx, "SELECT COUNT(*) AS count FROM connections", orm.WithResult(&result)); err != nil {
+		return 0, fmt.Errorf("failed to perform query: %w", err)
+	}
+
+	if len(result) != 1 {
+		return 0, fmt.Errorf("unexpected number of rows returned, expected 1 got %d", len(result))
+	}
+
+	return result[0].Count, nil
+}
+
+// Cleanup removes all connections that have ended before threshold.
+//
+// NOTE(ppacher): there is no easy way to get the number of removed
+// rows other than counting them in a first step. Though, that's
+// probably not worth the cylces...
+func (db *Database) Cleanup(ctx context.Context, threshold time.Time) (int, error) {
+	where := `WHERE ended IS NOT NULL
+			AND datetime(ended) < :threshold`
+	sql := "DELETE FROM connections " + where + ";"
+
+	args := orm.WithNamedArgs(map[string]interface{}{
+		":threshold": threshold,
+	})
+
+	var result []struct {
+		Count int `sqlite:"count"`
+	}
+	if err := db.Execute(
+		ctx,
+		"SELECT COUNT(*) AS count FROM connections "+where,
+		args,
+		orm.WithTransient(),
+		orm.WithResult(&result),
+	); err != nil {
+		return 0, fmt.Errorf("failed to perform query: %w", err)
+	}
+	if len(result) != 1 {
+		return 0, fmt.Errorf("unexpected number of rows, expected 1 got %d", len(result))
+	}
+
+	err := db.Execute(ctx, sql, args)
+	if err != nil {
+		return 0, err
+	}
+
+	return result[0].Count, nil
+}
+
+// dumpTo is a simple helper method that dumps all rows stored in the SQLite database
+// as JSON to w.
+// Any error aborts dumping rows and is returned.
+func (db *Database) dumpTo(ctx context.Context, w io.Writer) error {
+	db.l.Lock()
+	defer db.l.Unlock()
+
+	var conns []Conn
+	if err := sqlitex.ExecuteTransient(db.conn, "SELECT * FROM connections", &sqlitex.ExecOptions{
+		ResultFunc: func(stmt *sqlite.Stmt) error {
+			var c Conn
+			if err := orm.DecodeStmt(ctx, stmt, &c, orm.DefaultDecodeConfig); err != nil {
+				return err
+			}
+
+			conns = append(conns, c)
+			return nil
+		},
+	}); err != nil {
+		return err
+	}
+
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(conns)
+}
+
+// Save inserts the connection conn into the SQLite database. If conn
+// already exists the table row is updated instead.
+func (db *Database) Save(ctx context.Context, conn Conn) error {
+	connMap, err := orm.EncodeAsMap(ctx, conn, "", orm.DefaultEncodeConfig)
+	if err != nil {
+		return fmt.Errorf("failed to encode connection for SQL: %w", err)
+	}
+
+	columns := make([]string, 0, len(connMap))
+	placeholders := make([]string, 0, len(connMap))
+	values := make(map[string]interface{}, len(connMap))
+	updateSets := make([]string, 0, len(connMap))
+
+	for key, value := range connMap {
+		columns = append(columns, key)
+		placeholders = append(placeholders, ":"+key)
+		values[":"+key] = value
+		updateSets = append(updateSets, fmt.Sprintf("%s = :%s", key, key))
+	}
+
+	db.l.Lock()
+	defer db.l.Unlock()
+
+	// TODO(ppacher): make sure this one can be cached to speed up inserting
+	// and save some CPU cycles for the user
+	sql := fmt.Sprintf(
+		`INSERT INTO connections (%s)
+			VALUES(%s)
+			ON CONFLICT(id) DO UPDATE SET
+			%s
+		`,
+		strings.Join(columns, ", "),
+		strings.Join(placeholders, ", "),
+		strings.Join(updateSets, ", "),
+	)
+
+	if err := sqlitex.ExecuteTransient(db.conn, sql, &sqlitex.ExecOptions{
+		Named: values,
+		ResultFunc: func(stmt *sqlite.Stmt) error {
+			log.Errorf("netquery: got result statement with %d columns", stmt.ColumnCount())
+			return nil
+		},
+	}); err != nil {
+		log.Errorf("netquery: failed to execute: %s", err)
+		return err
+	}
+
+	return nil
+}
+
+// Close closes the underlying database connection. db should and cannot be
+// used after Close() has returned.
+func (db *Database) Close() error {
+	return db.conn.Close()
+}
diff --git a/netquery/manager.go b/netquery/manager.go
new file mode 100644
index 00000000..cd9b2335
--- /dev/null
+++ b/netquery/manager.go
@@ -0,0 +1,225 @@
+package netquery
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/runtime"
+	"github.com/safing/portmaster/network"
+)
+
+type (
+	// ConnectionStore describes the interface that is used by Manager
+	// to save new or updated connection objects.
+	// It is implemented by the *Database type of this package.
+	ConnectionStore interface {
+		// Save is called to perists the new or updated connection. If required,
+		// It's up the the implementation to figure out if the operation is an
+		// insert or an update.
+		// The ID of Conn is unique and can be trusted to never collide with other
+		// connections of the save device.
+		Save(context.Context, Conn) error
+	}
+
+	// Manager handles new and updated network.Connections feeds and persists them
+	// at a connection store.
+	// Manager also registers itself as a runtime database and pushes updates to
+	// connections using the local format.
+	// Users should use this update feed rather than the deprecated "network:" database.
+	Manager struct {
+		store      ConnectionStore
+		push       runtime.PushFunc
+		runtimeReg *runtime.Registry
+		pushPrefix string
+	}
+)
+
+// NewManager returns a new connection manager that persists all newly created or
+// updated connections at store.
+func NewManager(store ConnectionStore, pushPrefix string, reg *runtime.Registry) (*Manager, error) {
+	mng := &Manager{
+		store:      store,
+		runtimeReg: reg,
+		pushPrefix: pushPrefix,
+	}
+
+	push, err := reg.Register(pushPrefix, runtime.SimpleValueGetterFunc(mng.runtimeGet))
+	if err != nil {
+		return nil, err
+	}
+	mng.push = push
+
+	return mng, nil
+}
+
+func (mng *Manager) runtimeGet(keyOrPrefix string) ([]record.Record, error) {
+	// TODO(ppacher):
+	//		we don't yet support querying using the runtime database here ...
+	//		consider exposing connection from the database at least by ID.
+	//
+	// NOTE(ppacher):
+	//		for debugging purposes use RuntimeQueryRunner to execute plain
+	//		SQL queries against the database using portbase/database/runtime.
+	return nil, nil
+}
+
+// HandleFeed starts reading new and updated connections from feed and persists them
+// in the configured ConnectionStore. HandleFeed blocks until either ctx is cancelled
+// or feed is closed.
+// Any errors encountered when processing new or updated connections are logged but
+// otherwise ignored.
+// HandleFeed handles and persists updates one after each other! Depending on the system
+// load the user might want to use a buffered channel for feed.
+func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connection) {
+	// count the number of inserted rows for logging purposes.
+	//
+	// TODO(ppacher): how to handle the, though unlikely case, of a
+	// overflow to 0 here?
+	var count uint64
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+
+		case conn, ok := <-feed:
+			if !ok {
+				return
+			}
+
+			model, err := convertConnection(conn)
+			if err != nil {
+				log.Errorf("netquery: failed to convert connection %s to sqlite model: %w", conn.ID, err)
+
+				continue
+			}
+
+			log.Infof("netquery: persisting create/update to connection %s", conn.ID)
+
+			if err := mng.store.Save(ctx, *model); err != nil {
+				log.Errorf("netquery: failed to save connection %s in sqlite database: %w", conn.ID, err)
+
+				continue
+			}
+
+			// push an update for the connection
+			if err := mng.pushConnUpdate(ctx, *model); err != nil {
+				log.Errorf("netquery: failed to push update for conn %s via database system: %w", conn.ID, err)
+			}
+
+			count++
+
+			if count%20 == 0 {
+				log.Debugf("netquery: persisted %d connections so far", count)
+			}
+		}
+	}
+}
+
+func (mng *Manager) pushConnUpdate(ctx context.Context, conn Conn) error {
+	blob, err := json.Marshal(conn)
+	if err != nil {
+		return fmt.Errorf("failed to marshal connection: %w", err)
+	}
+
+	key := fmt.Sprintf("%s:%s%s", mng.runtimeReg.DatabaseName(), mng.pushPrefix, conn.ID)
+	wrapper, err := record.NewWrapper(
+		key,
+		new(record.Meta),
+		dsd.JSON,
+		blob,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create record wrapper: %w", err)
+	}
+
+	// FIXME(ppacher): it may happen that started != now for NEW connections.
+	// In that case we would push and UPD rather than NEW even if
+	// the connection is new ...
+	// Though, that's still better than always pushing NEW for existing
+	// connections.
+	// If we would use UnixNano() here chances would be even worse.
+	//
+	// Verify if the check in portbase/api/database.go is vulnerable
+	// to such timing issues in general.
+	wrapper.SetMeta(&record.Meta{
+		Created:  conn.Started.Unix(),
+		Modified: time.Now().Unix(),
+	})
+
+	mng.push(wrapper)
+	return nil
+}
+
+// convertConnection converts conn to the local representation used
+// to persist the information in SQLite. convertConnection attempts
+// to lock conn and may thus block for some time.
+func convertConnection(conn *network.Connection) (*Conn, error) {
+	conn.Lock()
+	defer conn.Unlock()
+
+	c := Conn{
+		ID:         genConnID(conn),
+		External:   conn.External,
+		IPVersion:  conn.IPVersion,
+		IPProtocol: conn.IPProtocol,
+		LocalIP:    conn.LocalIP.String(),
+		LocalPort:  conn.LocalPort,
+		Verdict:    conn.Verdict,
+		Started:    time.Unix(conn.Started, 0),
+		Tunneled:   conn.Tunneled,
+		Encrypted:  conn.Encrypted,
+		Internal:   conn.Internal,
+		Inbound:    conn.Inbound,
+		Type:       ConnectionTypeToString[conn.Type],
+	}
+
+	if conn.Ended > 0 {
+		ended := time.Unix(conn.Ended, 0)
+		c.Ended = &ended
+	}
+
+	extraData := map[string]interface{}{}
+
+	if conn.Entity != nil {
+		extraData["cname"] = conn.Entity.CNAME
+		extraData["blockedByLists"] = conn.Entity.BlockedByLists
+		extraData["blockedEntities"] = conn.Entity.BlockedEntities
+		extraData["reason"] = conn.Reason
+
+		c.RemoteIP = conn.Entity.IP.String()
+		c.RemotePort = conn.Entity.Port // FIXME(ppacher): or do we want DstPort() here?
+		c.Domain = conn.Entity.Domain
+		c.Country = conn.Entity.Country
+		c.ASN = conn.Entity.ASN
+		c.ASOwner = conn.Entity.ASOrg
+		c.Scope = conn.Entity.IPScope
+		if conn.Entity.Coordinates != nil {
+			c.Latitude = conn.Entity.Coordinates.Latitude
+			c.Longitude = conn.Entity.Coordinates.Longitude
+		}
+	}
+
+	// pre-compute the JSON blob for the extra data column
+	// and assign it.
+	extraDataBlob, err := json.Marshal(extraData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal extra data: %w", err)
+	}
+	c.ExtraData = extraDataBlob
+
+	return &c, nil
+}
+
+func genConnID(conn *network.Connection) string {
+	data := conn.ID + "-" + time.Unix(conn.Started, 0).String()
+	hash := sha256.Sum256([]byte(data))
+	return hex.EncodeToString(hash[:])
+}
diff --git a/netquery/module_api.go b/netquery/module_api.go
new file mode 100644
index 00000000..da4e7a1a
--- /dev/null
+++ b/netquery/module_api.go
@@ -0,0 +1,140 @@
+package netquery
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/query"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portbase/runtime"
+	"github.com/safing/portmaster/network"
+)
+
+func init() {
+	var (
+		module   *modules.Module
+		db       *database.Interface
+		sqlStore *Database
+		mng      *Manager
+	)
+
+	module = modules.Register(
+		"netquery",
+		/* Prepare Module */
+		func() error {
+			var err error
+
+			db = database.NewInterface(&database.Options{
+				Local:     true,
+				Internal:  true,
+				CacheSize: 0,
+			})
+
+			sqlStore, err = NewInMemory()
+			if err != nil {
+				return fmt.Errorf("failed to create in-memory database: %w", err)
+			}
+
+			mng, err = NewManager(sqlStore, "netquery/updates/", runtime.DefaultRegistry)
+			if err != nil {
+				return fmt.Errorf("failed to create manager: %w", err)
+			}
+
+			return nil
+		},
+		/* Start Module */
+		func() error {
+			ch := make(chan *network.Connection, 100)
+
+			module.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error {
+				sub, err := db.Subscribe(query.New("network:"))
+				if err != nil {
+					return fmt.Errorf("failed to subscribe to network tree: %w", err)
+				}
+				defer sub.Cancel()
+
+				for {
+					select {
+					case <-ctx.Done():
+						return nil
+					case rec, ok := <-sub.Feed:
+						if !ok {
+							return nil
+						}
+
+						conn, ok := rec.(*network.Connection)
+						if !ok {
+							// This is fine as we also receive process updates on
+							// this channel.
+							continue
+						}
+
+						ch <- conn
+					}
+				}
+			})
+
+			module.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error {
+				defer close(ch)
+
+				mng.HandleFeed(ctx, ch)
+				return nil
+			})
+
+			module.StartWorker("netquery-row-cleaner", func(ctx context.Context) error {
+				for {
+					select {
+					case <-ctx.Done():
+						return nil
+					case <-time.After(10 * time.Second):
+						count, err := sqlStore.Cleanup(ctx, time.Now().Add(-5*time.Minute))
+						if err != nil {
+							log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+						} else {
+							log.Infof("netquery: successfully removed %d old rows", count)
+						}
+					}
+				}
+			})
+
+			module.StartWorker("netquery-row-counter", func(ctx context.Context) error {
+				for {
+					select {
+					case <-ctx.Done():
+						return nil
+					case <-time.After(5 * time.Second):
+						count, err := sqlStore.CountRows(ctx)
+						if err != nil {
+							log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+						} else {
+							log.Infof("netquery: currently holding %d rows in memory", count)
+						}
+
+						/*
+							if err := sqlStore.dumpTo(ctx, os.Stderr); err != nil {
+								log.Errorf("netquery: failed to dump sqlite memory content: %w", err)
+							}
+						*/
+					}
+				}
+			})
+
+			// for debugging, we provide a simple direct SQL query interface using
+			// the runtime database
+			_, err := NewRuntimeQueryRunner(sqlStore, "netquery/query/", runtime.DefaultRegistry)
+			if err != nil {
+				return fmt.Errorf("failed to set up runtime SQL query runner: %w", err)
+			}
+
+			return nil
+		},
+		nil,
+		"network",
+		"database",
+	)
+
+	module.Enable()
+}
diff --git a/netquery/runtime_query_runner.go b/netquery/runtime_query_runner.go
new file mode 100644
index 00000000..43f58539
--- /dev/null
+++ b/netquery/runtime_query_runner.go
@@ -0,0 +1,77 @@
+package netquery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/runtime"
+	"github.com/safing/portmaster/netquery/orm"
+)
+
+// RuntimeQueryRunner provides a simple interface for the runtime database
+// that allows direct SQL queries to be performed against db.
+// Each resulting row of that query are marshaled as map[string]interface{}
+// and returned as a single record to the caller.
+//
+// Using portbase/database#Query is not possible because portbase/database will
+// complain about the SQL query being invalid. To work around that issue,
+// RuntimeQueryRunner uses a 'GET key' request where the SQL query is embedded into
+// the record key.
+type RuntimeQueryRunner struct {
+	db        *Database
+	reg       *runtime.Registry
+	keyPrefix string
+}
+
+// NewRuntimeQueryRunner returns a new runtime SQL query runner that parses
+// and serves SQL queries form GET <prefix>/<plain sql query> requests.
+func NewRuntimeQueryRunner(db *Database, prefix string, reg *runtime.Registry) (*RuntimeQueryRunner, error) {
+	runner := &RuntimeQueryRunner{
+		db:        db,
+		reg:       reg,
+		keyPrefix: prefix,
+	}
+
+	if _, err := reg.Register(prefix, runtime.SimpleValueGetterFunc(runner.get)); err != nil {
+		return nil, fmt.Errorf("failed to register runtime value provider: %w", err)
+	}
+
+	return runner, nil
+}
+
+func (runner *RuntimeQueryRunner) get(keyOrPrefix string) ([]record.Record, error) {
+	query := strings.TrimPrefix(
+		keyOrPrefix,
+		runner.keyPrefix,
+	)
+
+	log.Infof("netquery: executing custom SQL query: %q", query)
+
+	var result []map[string]interface{}
+	if err := runner.db.Execute(context.Background(), query, orm.WithResult(&result)); err != nil {
+		return nil, fmt.Errorf("failed to perform query %q: %w", query, err)
+	}
+
+	// we need to wrap the result slice into a map as portbase/database attempts
+	// to inject a _meta field.
+	blob, err := json.Marshal(map[string]interface{}{
+		"result": result,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal result: %w", err)
+	}
+
+	key := fmt.Sprintf("%s:%s", runner.reg.DatabaseName(), keyOrPrefix)
+	wrapper, err := record.NewWrapper(key, nil, dsd.JSON, blob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create record wrapper: %w", err)
+	}
+	wrapper.SetMeta(new(record.Meta))
+
+	return []record.Record{wrapper}, nil
+}

From 021c9b178cf8892cbd568fffe7624a07c847e5c3 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Thu, 17 Mar 2022 09:57:10 +0100
Subject: [PATCH 54/86] Add documentation to query runner

---
 netquery/orm/query_runner.go | 41 ++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go
index d94e62db..6de0c66a 100644
--- a/netquery/orm/query_runner.go
+++ b/netquery/orm/query_runner.go
@@ -10,6 +10,8 @@ import (
 )
 
 type (
+	// QueryOption can be specified at RunQuery to alter the behavior
+	// of the executed query.
 	QueryOption func(opts *queryOpts)
 
 	queryOpts struct {
@@ -21,36 +23,75 @@ type (
 	}
 )
 
+// WithTransient marks the query as transient.
+//
+// Transient queries will not be cached for later
+// re-use after they have been prepared.
 func WithTransient() QueryOption {
 	return func(opts *queryOpts) {
 		opts.Transient = true
 	}
 }
 
+// WithArgs adds a list of arguments for the query. Arguments
+// are applied in order.
+//
+// See SQL Language Expression documentation of SQLite for
+// details: https://sqlite.org/lang_expr.html
 func WithArgs(args ...interface{}) QueryOption {
 	return func(opts *queryOpts) {
 		opts.Args = args
 	}
 }
 
+// WithNamedArgs adds args to the query. The query must used
+// named argument placeholders. According to the SQLite spec,
+// arguments must either start with ':', '@' or '$'.
+//
+// See SQL Language Expression documentation of SQLite for
+// details: https://sqlite.org/lang_expr.html
 func WithNamedArgs(args map[string]interface{}) QueryOption {
 	return func(opts *queryOpts) {
 		opts.NamedArgs = args
 	}
 }
 
+// WithResult sets the result receiver. result is expected to
+// be a pointer to a slice of struct or map types.
+//
+// For decoding DecodeStmt is used to decode each
+// row into a new slice element. It thus supports special values
+// like time.Time. See DecodeStmt() and WithDecodeConfig() for
+// more information.
 func WithResult(result interface{}) QueryOption {
 	return func(opts *queryOpts) {
 		opts.Result = result
 	}
 }
 
+// WithDecodeConfig configures the DecodeConfig to use when
+// calling DecodeStmt to decode each row into the result slice.
+//
+// If not specified, DefaultDecodeConfig will be used.
 func WithDecodeConfig(cfg DecodeConfig) QueryOption {
 	return func(opts *queryOpts) {
 		opts.DecodeConfig = cfg
 	}
 }
 
+// RunQuery executes the query stored in sql against the databased opened in
+// conn. Please refer to the documentation of QueryOption, especially WithResult()
+// for more information on how to retrieve the resulting rows.
+//
+// Example:
+//
+//		var result []struct{
+//			Count int `sqlite:"rowCount"`
+//		}
+//
+//		err := RunQuery(ctx, conn, "SELECT COUNT(*) AS rowCount FROM table", WithResult(&result))
+//		fmt.Println(result[0].Count)
+//
 func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...QueryOption) error {
 	args := queryOpts{
 		DecodeConfig: DefaultDecodeConfig,

From 976c0a702e2d129bedb34dec087c4656f027fdcf Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Thu, 17 Mar 2022 09:57:48 +0100
Subject: [PATCH 55/86] Add netquery module to main

---
 cmds/portmaster-core/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index b7f1d1ec..edfbe197 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -13,6 +13,7 @@ import ( //nolint:gci,nolintlint
 	_ "github.com/safing/portmaster/core"
 	_ "github.com/safing/portmaster/firewall"
 	_ "github.com/safing/portmaster/nameserver"
+	_ "github.com/safing/portmaster/netquery"
 	_ "github.com/safing/portmaster/ui"
 	_ "github.com/safing/spn/captain"
 )

From c2d2064ec83bf1bc46b85afb094edeeb87201de7 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Thu, 17 Mar 2022 14:28:01 +0100
Subject: [PATCH 56/86] Add and update netquery code based on review
 discussions

---
 cmds/portmaster-core/main.go   |   1 -
 core/core.go                   |   6 +-
 netquery/database.go           |   2 +-
 netquery/manager.go            |  27 ++--
 netquery/module_api.go         | 245 +++++++++++++++++----------------
 netquery/orm/encoder.go        |  83 ++++++++++-
 netquery/orm/encoder_test.go   | 100 +++++++++++++-
 netquery/orm/schema_builder.go |   4 +
 network/clean.go               |   4 +-
 9 files changed, 324 insertions(+), 148 deletions(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index edfbe197..b7f1d1ec 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -13,7 +13,6 @@ import ( //nolint:gci,nolintlint
 	_ "github.com/safing/portmaster/core"
 	_ "github.com/safing/portmaster/firewall"
 	_ "github.com/safing/portmaster/nameserver"
-	_ "github.com/safing/portmaster/netquery"
 	_ "github.com/safing/portmaster/ui"
 	_ "github.com/safing/spn/captain"
 )
diff --git a/core/core.go b/core/core.go
index 80bd6343..ec2431b6 100644
--- a/core/core.go
+++ b/core/core.go
@@ -7,10 +7,12 @@ import (
 
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
+	"github.com/safing/portmaster/updates"
+
 	_ "github.com/safing/portmaster/netenv"
+	_ "github.com/safing/portmaster/netquery"
 	_ "github.com/safing/portmaster/status"
 	_ "github.com/safing/portmaster/ui"
-	"github.com/safing/portmaster/updates"
 )
 
 const (
@@ -25,7 +27,7 @@ var (
 )
 
 func init() {
-	module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "interception", "compat")
+	module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "interception", "compat", "netquery")
 	subsystems.Register(
 		"core",
 		"Core",
diff --git a/netquery/database.go b/netquery/database.go
index 7485f19c..cf5450a0 100644
--- a/netquery/database.go
+++ b/netquery/database.go
@@ -244,7 +244,7 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error {
 // Save inserts the connection conn into the SQLite database. If conn
 // already exists the table row is updated instead.
 func (db *Database) Save(ctx context.Context, conn Conn) error {
-	connMap, err := orm.EncodeAsMap(ctx, conn, "", orm.DefaultEncodeConfig)
+	connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig)
 	if err != nil {
 		return fmt.Errorf("failed to encode connection for SQL: %w", err)
 	}
diff --git a/netquery/manager.go b/netquery/manager.go
index cd9b2335..647e82ed 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -109,8 +109,13 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 				continue
 			}
 
+			// we clone the record metadata from the connection
+			// into the new model so the portbase/database layer
+			// can handle NEW/UPDATE correctly.
+			cloned := conn.Meta().Duplicate()
+
 			// push an update for the connection
-			if err := mng.pushConnUpdate(ctx, *model); err != nil {
+			if err := mng.pushConnUpdate(ctx, *cloned, *model); err != nil {
 				log.Errorf("netquery: failed to push update for conn %s via database system: %w", conn.ID, err)
 			}
 
@@ -123,7 +128,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 	}
 }
 
-func (mng *Manager) pushConnUpdate(ctx context.Context, conn Conn) error {
+func (mng *Manager) pushConnUpdate(ctx context.Context, meta record.Meta, conn Conn) error {
 	blob, err := json.Marshal(conn)
 	if err != nil {
 		return fmt.Errorf("failed to marshal connection: %w", err)
@@ -132,7 +137,7 @@ func (mng *Manager) pushConnUpdate(ctx context.Context, conn Conn) error {
 	key := fmt.Sprintf("%s:%s%s", mng.runtimeReg.DatabaseName(), mng.pushPrefix, conn.ID)
 	wrapper, err := record.NewWrapper(
 		key,
-		new(record.Meta),
+		&meta,
 		dsd.JSON,
 		blob,
 	)
@@ -140,20 +145,6 @@ func (mng *Manager) pushConnUpdate(ctx context.Context, conn Conn) error {
 		return fmt.Errorf("failed to create record wrapper: %w", err)
 	}
 
-	// FIXME(ppacher): it may happen that started != now for NEW connections.
-	// In that case we would push and UPD rather than NEW even if
-	// the connection is new ...
-	// Though, that's still better than always pushing NEW for existing
-	// connections.
-	// If we would use UnixNano() here chances would be even worse.
-	//
-	// Verify if the check in portbase/api/database.go is vulnerable
-	// to such timing issues in general.
-	wrapper.SetMeta(&record.Meta{
-		Created:  conn.Started.Unix(),
-		Modified: time.Now().Unix(),
-	})
-
 	mng.push(wrapper)
 	return nil
 }
@@ -195,7 +186,7 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		extraData["reason"] = conn.Reason
 
 		c.RemoteIP = conn.Entity.IP.String()
-		c.RemotePort = conn.Entity.Port // FIXME(ppacher): or do we want DstPort() here?
+		c.RemotePort = conn.Entity.Port
 		c.Domain = conn.Entity.Domain
 		c.Country = conn.Entity.Country
 		c.ASN = conn.Entity.ASN
diff --git a/netquery/module_api.go b/netquery/module_api.go
index da4e7a1a..6bc9cb4f 100644
--- a/netquery/module_api.go
+++ b/netquery/module_api.go
@@ -13,128 +13,135 @@ import (
 	"github.com/safing/portmaster/network"
 )
 
+type Module struct {
+	*modules.Module
+
+	db       *database.Interface
+	sqlStore *Database
+	mng      *Manager
+	feed     chan *network.Connection
+}
+
 func init() {
-	var (
-		module   *modules.Module
-		db       *database.Interface
-		sqlStore *Database
-		mng      *Manager
-	)
-
-	module = modules.Register(
+	mod := new(Module)
+	mod.Module = modules.Register(
 		"netquery",
-		/* Prepare Module */
-		func() error {
-			var err error
-
-			db = database.NewInterface(&database.Options{
-				Local:     true,
-				Internal:  true,
-				CacheSize: 0,
-			})
-
-			sqlStore, err = NewInMemory()
-			if err != nil {
-				return fmt.Errorf("failed to create in-memory database: %w", err)
-			}
-
-			mng, err = NewManager(sqlStore, "netquery/updates/", runtime.DefaultRegistry)
-			if err != nil {
-				return fmt.Errorf("failed to create manager: %w", err)
-			}
-
-			return nil
-		},
-		/* Start Module */
-		func() error {
-			ch := make(chan *network.Connection, 100)
-
-			module.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error {
-				sub, err := db.Subscribe(query.New("network:"))
-				if err != nil {
-					return fmt.Errorf("failed to subscribe to network tree: %w", err)
-				}
-				defer sub.Cancel()
-
-				for {
-					select {
-					case <-ctx.Done():
-						return nil
-					case rec, ok := <-sub.Feed:
-						if !ok {
-							return nil
-						}
-
-						conn, ok := rec.(*network.Connection)
-						if !ok {
-							// This is fine as we also receive process updates on
-							// this channel.
-							continue
-						}
-
-						ch <- conn
-					}
-				}
-			})
-
-			module.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error {
-				defer close(ch)
-
-				mng.HandleFeed(ctx, ch)
-				return nil
-			})
-
-			module.StartWorker("netquery-row-cleaner", func(ctx context.Context) error {
-				for {
-					select {
-					case <-ctx.Done():
-						return nil
-					case <-time.After(10 * time.Second):
-						count, err := sqlStore.Cleanup(ctx, time.Now().Add(-5*time.Minute))
-						if err != nil {
-							log.Errorf("netquery: failed to count number of rows in memory: %w", err)
-						} else {
-							log.Infof("netquery: successfully removed %d old rows", count)
-						}
-					}
-				}
-			})
-
-			module.StartWorker("netquery-row-counter", func(ctx context.Context) error {
-				for {
-					select {
-					case <-ctx.Done():
-						return nil
-					case <-time.After(5 * time.Second):
-						count, err := sqlStore.CountRows(ctx)
-						if err != nil {
-							log.Errorf("netquery: failed to count number of rows in memory: %w", err)
-						} else {
-							log.Infof("netquery: currently holding %d rows in memory", count)
-						}
-
-						/*
-							if err := sqlStore.dumpTo(ctx, os.Stderr); err != nil {
-								log.Errorf("netquery: failed to dump sqlite memory content: %w", err)
-							}
-						*/
-					}
-				}
-			})
-
-			// for debugging, we provide a simple direct SQL query interface using
-			// the runtime database
-			_, err := NewRuntimeQueryRunner(sqlStore, "netquery/query/", runtime.DefaultRegistry)
-			if err != nil {
-				return fmt.Errorf("failed to set up runtime SQL query runner: %w", err)
-			}
-
-			return nil
-		},
-		nil,
+		mod.Prepare,
+		mod.Start,
+		mod.Stop,
 		"network",
 		"database",
 	)
-
-	module.Enable()
+}
+
+func (m *Module) Prepare() error {
+	var err error
+
+	m.db = database.NewInterface(&database.Options{
+		Local:     true,
+		Internal:  true,
+		CacheSize: 0,
+	})
+
+	m.sqlStore, err = NewInMemory()
+	if err != nil {
+		return fmt.Errorf("failed to create in-memory database: %w", err)
+	}
+
+	m.mng, err = NewManager(m.sqlStore, "netquery/data/", runtime.DefaultRegistry)
+	if err != nil {
+		return fmt.Errorf("failed to create manager: %w", err)
+	}
+
+	m.feed = make(chan *network.Connection, 1000)
+
+	return nil
+}
+
+func (mod *Module) Start() error {
+	mod.StartServiceWorker("netquery-feeder", time.Second, func(ctx context.Context) error {
+		sub, err := mod.db.Subscribe(query.New("network:"))
+		if err != nil {
+			return fmt.Errorf("failed to subscribe to network tree: %w", err)
+		}
+		defer sub.Cancel()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return nil
+			case rec, ok := <-sub.Feed:
+				if !ok {
+					return nil
+				}
+
+				conn, ok := rec.(*network.Connection)
+				if !ok {
+					// This is fine as we also receive process updates on
+					// this channel.
+					continue
+				}
+
+				mod.feed <- conn
+			}
+		}
+	})
+
+	mod.StartServiceWorker("netquery-persister", time.Second, func(ctx context.Context) error {
+		mod.mng.HandleFeed(ctx, mod.feed)
+		return nil
+	})
+
+	mod.StartServiceWorker("netquery-row-cleaner", time.Second, func(ctx context.Context) error {
+		for {
+			select {
+			case <-ctx.Done():
+				return nil
+			case <-time.After(10 * time.Second):
+				count, err := mod.sqlStore.Cleanup(ctx, time.Now().Add(-network.DeleteConnsAfterEndedThreshold))
+				if err != nil {
+					log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+				} else {
+					log.Infof("netquery: successfully removed %d old rows", count)
+				}
+			}
+		}
+	})
+
+	mod.StartWorker("netquery-row-counter", func(ctx context.Context) error {
+		for {
+			select {
+			case <-ctx.Done():
+				return nil
+			case <-time.After(5 * time.Second):
+				count, err := mod.sqlStore.CountRows(ctx)
+				if err != nil {
+					log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+				} else {
+					log.Infof("netquery: currently holding %d rows in memory", count)
+				}
+
+				/*
+					if err := sqlStore.dumpTo(ctx, os.Stderr); err != nil {
+						log.Errorf("netquery: failed to dump sqlite memory content: %w", err)
+					}
+				*/
+			}
+		}
+	})
+
+	// for debugging, we provide a simple direct SQL query interface using
+	// the runtime database
+	_, err := NewRuntimeQueryRunner(mod.sqlStore, "netquery/query/", runtime.DefaultRegistry)
+	if err != nil {
+		return fmt.Errorf("failed to set up runtime SQL query runner: %w", err)
+	}
+
+	return nil
+}
+
+func (mod *Module) Stop() error {
+	close(mod.feed)
+
+	return nil
 }
diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go
index c2d9a62c..fc4e772c 100644
--- a/netquery/orm/encoder.go
+++ b/netquery/orm/encoder.go
@@ -17,10 +17,10 @@ type (
 	}
 )
 
-// EncodeAsMap returns a map that contains the value of each struct field of
+// ToParamMap returns a map that contains the sqlite compatible value of each struct field of
 // r using the sqlite column name as a map key. It either uses the name of the
 // exported struct field or the value of the "sqlite" tag.
-func EncodeAsMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig) (map[string]interface{}, error) {
+func ToParamMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig) (map[string]interface{}, error) {
 	// make sure we work on a struct type
 	val := reflect.Indirect(reflect.ValueOf(r))
 	if val.Kind() != reflect.Struct {
@@ -38,12 +38,20 @@ func EncodeAsMap(ctx context.Context, r interface{}, keyPrefix string, cfg Encod
 			continue
 		}
 
-		colDev, err := getColumnDef(fieldType)
+		colDef, err := getColumnDef(fieldType)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get column definition for %s: %w", fieldType.Name, err)
 		}
 
-		x, found, err := runEncodeHooks(colDev, fieldType.Type, field, cfg.EncodeHooks)
+		x, found, err := runEncodeHooks(
+			colDef,
+			fieldType.Type,
+			field,
+			append(
+				cfg.EncodeHooks,
+				encodeBasic(),
+			),
+		)
 		if err != nil {
 			return nil, fmt.Errorf("failed to run encode hooks: %w", err)
 		}
@@ -61,6 +69,69 @@ func EncodeAsMap(ctx context.Context, r interface{}, keyPrefix string, cfg Encod
 	return res, nil
 }
 
+func EncodeValue(ctx context.Context, colDef *ColumnDef, val interface{}, cfg EncodeConfig) (interface{}, error) {
+	fieldValue := reflect.ValueOf(val)
+	fieldType := reflect.TypeOf(val)
+
+	x, found, err := runEncodeHooks(
+		colDef,
+		fieldType,
+		fieldValue,
+		append(
+			cfg.EncodeHooks,
+			encodeBasic(),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to run encode hooks: %w", err)
+	}
+
+	if !found {
+		if reflect.Indirect(fieldValue).IsValid() {
+			x = reflect.Indirect(fieldValue).Interface()
+		}
+	}
+
+	return x, nil
+}
+
+func encodeBasic() EncodeFunc {
+	return func(col *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) {
+		kind := valType.Kind()
+		if kind == reflect.Ptr {
+			valType = valType.Elem()
+			kind = valType.Kind()
+
+			if val.IsNil() {
+				return nil, true, nil
+			}
+
+			val = val.Elem()
+		}
+
+		switch normalizeKind(kind) {
+		case reflect.String,
+			reflect.Float64,
+			reflect.Bool,
+			reflect.Int,
+			reflect.Uint:
+			// sqlite package handles conversion of those types
+			// already
+			return val.Interface(), true, nil
+
+		case reflect.Slice:
+			if valType.Elem().Kind() == reflect.Uint8 {
+				// this is []byte
+				return val.Interface(), true, nil
+			}
+			fallthrough
+
+		default:
+			return nil, false, fmt.Errorf("cannot convert value of kind %s for use in SQLite", kind)
+		}
+	}
+}
+
 func DatetimeEncoder(loc *time.Location) EncodeFunc {
 	return func(colDev *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) {
 		// if fieldType holds a pointer we need to dereference the value
@@ -103,6 +174,10 @@ func DatetimeEncoder(loc *time.Location) EncodeFunc {
 }
 
 func runEncodeHooks(colDev *ColumnDef, valType reflect.Type, val reflect.Value, hooks []EncodeFunc) (interface{}, bool, error) {
+	if valType == nil {
+		return nil, true, nil
+	}
+
 	for _, fn := range hooks {
 		res, end, err := fn(colDev, valType, val)
 		if err != nil {
diff --git a/netquery/orm/encoder_test.go b/netquery/orm/encoder_test.go
index 056bf953..8b802a78 100644
--- a/netquery/orm/encoder_test.go
+++ b/netquery/orm/encoder_test.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"zombiezen.com/go/sqlite"
 )
 
 func Test_EncodeAsMap(t *testing.T) {
@@ -118,9 +119,106 @@ func Test_EncodeAsMap(t *testing.T) {
 		t.Run(c.Desc, func(t *testing.T) {
 			// t.Parallel()
 
-			res, err := EncodeAsMap(ctx, c.Input, "", DefaultEncodeConfig)
+			res, err := ToParamMap(ctx, c.Input, "", DefaultEncodeConfig)
 			assert.NoError(t, err)
 			assert.Equal(t, c.Expected, res)
 		})
 	}
 }
+
+func Test_EncodeValue(t *testing.T) {
+	ctx := context.TODO()
+	refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC)
+
+	cases := []struct {
+		Desc   string
+		Column ColumnDef
+		Input  interface{}
+		Output interface{}
+	}{
+		{
+			"Special value time.Time as text",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			refTime,
+			refTime.Format(sqliteTimeFormat),
+		},
+		{
+			"Special value time.Time as unix-epoch",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeInteger,
+			},
+			refTime,
+			refTime.Unix(),
+		},
+		{
+			"Special value time.Time as unixnano-epoch",
+			ColumnDef{
+				IsTime:   true,
+				Type:     sqlite.TypeInteger,
+				UnixNano: true,
+			},
+			refTime,
+			refTime.UnixNano(),
+		},
+		{
+			"Special value zero time",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			time.Time{},
+			nil,
+		},
+		{
+			"Special value zero time pointer",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			new(time.Time),
+			nil,
+		},
+		{
+			"Special value *time.Time as text",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			&refTime,
+			refTime.Format(sqliteTimeFormat),
+		},
+		{
+			"Special value untyped nil",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			nil,
+			nil,
+		},
+		{
+			"Special value typed nil",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			(*time.Time)(nil),
+			nil,
+		},
+	}
+
+	for idx := range cases {
+		c := cases[idx]
+		t.Run(c.Desc, func(t *testing.T) {
+			// t.Parallel()
+
+			res, err := EncodeValue(ctx, &c.Column, c.Input, DefaultEncodeConfig)
+			assert.NoError(t, err)
+			assert.Equal(t, c.Output, res)
+		})
+	}
+}
diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go
index aa57e21a..9289b06d 100644
--- a/netquery/orm/schema_builder.go
+++ b/netquery/orm/schema_builder.go
@@ -18,6 +18,7 @@ var (
 	TagUnixNano          = "unixnano"
 	TagPrimaryKey        = "primary"
 	TagAutoIncrement     = "autoincrement"
+	TagTime              = "time"
 	TagNotNull           = "not-null"
 	TagNullable          = "nullable"
 	TagTypeInt           = "integer"
@@ -48,6 +49,7 @@ type (
 		PrimaryKey    bool
 		AutoIncrement bool
 		UnixNano      bool
+		IsTime        bool
 	}
 )
 
@@ -188,6 +190,8 @@ func applyStructFieldTag(fieldType reflect.StructField, def *ColumnDef) error {
 				def.Nullable = true
 			case TagUnixNano:
 				def.UnixNano = true
+			case TagTime:
+				def.IsTime = true
 
 			// basic column types
 			case TagTypeInt:
diff --git a/network/clean.go b/network/clean.go
index 608e2c88..6ff1eb38 100644
--- a/network/clean.go
+++ b/network/clean.go
@@ -12,7 +12,7 @@ import (
 
 const (
 	cleanerTickDuration            = 5 * time.Second
-	deleteConnsAfterEndedThreshold = 10 * time.Minute
+	DeleteConnsAfterEndedThreshold = 10 * time.Minute
 )
 
 func connectionCleaner(ctx context.Context) error {
@@ -41,7 +41,7 @@ func cleanConnections() (activePIDs map[int]struct{}) {
 	_ = module.RunMediumPriorityMicroTask(&name, func(ctx context.Context) error {
 		now := time.Now().UTC()
 		nowUnix := now.Unix()
-		deleteOlderThan := now.Add(-deleteConnsAfterEndedThreshold).Unix()
+		deleteOlderThan := now.Add(-DeleteConnsAfterEndedThreshold).Unix()
 
 		// network connections
 		for _, conn := range conns.clone() {

From e21eb16a6b827ca21ffb19961b03d0a4e9bb4ba1 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Mon, 21 Mar 2022 19:17:36 +0100
Subject: [PATCH 57/86] Use new(record.Meta) instead of a dedicated call to
 SetMeta

---
 netquery/runtime_query_runner.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/netquery/runtime_query_runner.go b/netquery/runtime_query_runner.go
index 43f58539..3b443ec5 100644
--- a/netquery/runtime_query_runner.go
+++ b/netquery/runtime_query_runner.go
@@ -66,12 +66,12 @@ func (runner *RuntimeQueryRunner) get(keyOrPrefix string) ([]record.Record, erro
 		return nil, fmt.Errorf("failed to marshal result: %w", err)
 	}
 
+	// construct a new record wrapper that uses the already prepared JSON blob.
 	key := fmt.Sprintf("%s:%s", runner.reg.DatabaseName(), keyOrPrefix)
-	wrapper, err := record.NewWrapper(key, nil, dsd.JSON, blob)
+	wrapper, err := record.NewWrapper(key, new(record.Meta), dsd.JSON, blob)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create record wrapper: %w", err)
 	}
-	wrapper.SetMeta(new(record.Meta))
 
 	return []record.Record{wrapper}, nil
 }

From d098f1c1379021fdab272d174d9bee77cf4e09d6 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 26 Apr 2022 14:59:27 +0200
Subject: [PATCH 58/86] Add support for new query API

---
 netquery/database.go           |  27 +-
 netquery/manager.go            |   8 +-
 netquery/module_api.go         |  26 +-
 netquery/orm/decoder.go        |   4 +-
 netquery/orm/encoder.go        |  72 +++--
 netquery/orm/encoder_test.go   |  50 +++-
 netquery/orm/schema_builder.go |   9 +
 netquery/query.go              | 464 +++++++++++++++++++++++++++++++++
 netquery/query_handler.go      | 293 +++++++++++++++++++++
 netquery/query_test.go         | 244 +++++++++++++++++
 10 files changed, 1154 insertions(+), 43 deletions(-)
 create mode 100644 netquery/query.go
 create mode 100644 netquery/query_handler.go
 create mode 100644 netquery/query_test.go

diff --git a/netquery/database.go b/netquery/database.go
index cf5450a0..25f9e571 100644
--- a/netquery/database.go
+++ b/netquery/database.go
@@ -44,6 +44,8 @@ type (
 	// are actually supposed to do.
 	//
 	Database struct {
+		Schema *orm.TableSchema
+
 		l    sync.Mutex
 		conn *sqlite.Conn
 	}
@@ -62,6 +64,8 @@ type (
 		// as long as the connection is still active and might be, although unlikely,
 		// reused afterwards.
 		ID         string            `sqlite:"id,primary"`
+		ProfileID  string            `sqlite:"profile"`
+		Path       string            `sqlite:"path"`
 		Type       string            `sqlite:"type,varchar(8)"`
 		External   bool              `sqlite:"external"`
 		IPVersion  packet.IPVersion  `sqlite:"ip_version"`
@@ -78,8 +82,8 @@ type (
 		Longitude  float64           `sqlite:"longitude"`
 		Scope      netutils.IPScope  `sqlite:"scope"`
 		Verdict    network.Verdict   `sqlite:"verdict"`
-		Started    time.Time         `sqlite:"started,text"`
-		Ended      *time.Time        `sqlite:"ended,text"`
+		Started    time.Time         `sqlite:"started,text,time"`
+		Ended      *time.Time        `sqlite:"ended,text,time"`
 		Tunneled   bool              `sqlite:"tunneled"`
 		Encrypted  bool              `sqlite:"encrypted"`
 		Internal   bool              `sqlite:"internal"`
@@ -107,7 +111,15 @@ func New(path string) (*Database, error) {
 		return nil, fmt.Errorf("failed to open sqlite at %s: %w", path, err)
 	}
 
-	return &Database{conn: c}, nil
+	schema, err := orm.GenerateTableSchema("connections", Conn{})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Database{
+		Schema: schema,
+		conn:   c,
+	}, nil
 }
 
 // NewInMemory is like New but creates a new in-memory database and
@@ -133,13 +145,8 @@ func NewInMemory() (*Database, error) {
 // any data-migrations. Once the history module is implemented this should
 // become/use a full migration system -- use zombiezen.com/go/sqlite/sqlitemigration
 func (db *Database) ApplyMigrations() error {
-	schema, err := orm.GenerateTableSchema("connections", Conn{})
-	if err != nil {
-		return fmt.Errorf("failed to generate table schema for conncetions: %w", err)
-	}
-
 	// get the create-table SQL statement from the infered schema
-	sql := schema.CreateStatement(false)
+	sql := db.Schema.CreateStatement(false)
 
 	// execute the SQL
 	if err := sqlitex.ExecuteTransient(db.conn, sql, nil); err != nil {
@@ -284,7 +291,7 @@ func (db *Database) Save(ctx context.Context, conn Conn) error {
 			return nil
 		},
 	}); err != nil {
-		log.Errorf("netquery: failed to execute: %s", err)
+		log.Errorf("netquery: failed to execute:\n\t%q\n\treturned error was: %s\n\tparameters: %+v", sql, err, values)
 		return err
 	}
 
diff --git a/netquery/manager.go b/netquery/manager.go
index 647e82ed..16a78021 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -96,7 +96,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 
 			model, err := convertConnection(conn)
 			if err != nil {
-				log.Errorf("netquery: failed to convert connection %s to sqlite model: %w", conn.ID, err)
+				log.Errorf("netquery: failed to convert connection %s to sqlite model: %s", conn.ID, err)
 
 				continue
 			}
@@ -104,7 +104,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 			log.Infof("netquery: persisting create/update to connection %s", conn.ID)
 
 			if err := mng.store.Save(ctx, *model); err != nil {
-				log.Errorf("netquery: failed to save connection %s in sqlite database: %w", conn.ID, err)
+				log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err)
 
 				continue
 			}
@@ -116,7 +116,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 
 			// push an update for the connection
 			if err := mng.pushConnUpdate(ctx, *cloned, *model); err != nil {
-				log.Errorf("netquery: failed to push update for conn %s via database system: %w", conn.ID, err)
+				log.Errorf("netquery: failed to push update for conn %s via database system: %s", conn.ID, err)
 			}
 
 			count++
@@ -170,6 +170,8 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		Internal:   conn.Internal,
 		Inbound:    conn.Inbound,
 		Type:       ConnectionTypeToString[conn.Type],
+		ProfileID:  conn.ProcessContext.ProfileName,
+		Path:       conn.ProcessContext.BinaryPath,
 	}
 
 	if conn.Ended > 0 {
diff --git a/netquery/module_api.go b/netquery/module_api.go
index 6bc9cb4f..a344ec3b 100644
--- a/netquery/module_api.go
+++ b/netquery/module_api.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/config"
 	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/log"
@@ -29,6 +31,7 @@ func init() {
 		mod.Prepare,
 		mod.Start,
 		mod.Stop,
+		"api",
 		"network",
 		"database",
 	)
@@ -55,6 +58,25 @@ func (m *Module) Prepare() error {
 
 	m.feed = make(chan *network.Connection, 1000)
 
+	queryHander := &QueryHandler{
+		Database:  m.sqlStore,
+		IsDevMode: config.Concurrent.GetAsBool(config.CfgDevModeKey, false),
+	}
+
+	// FIXME(ppacher): use appropriate permissions for this
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:        "netquery/query",
+		MimeType:    "application/json",
+		Read:        api.PermitAnyone,
+		Write:       api.PermitAnyone,
+		BelongsTo:   m.Module,
+		HandlerFunc: queryHander.ServeHTTP,
+		Name:        "Query In-Memory Database",
+		Description: "Query the in-memory sqlite database",
+	}); err != nil {
+		return fmt.Errorf("failed to register API endpoint: %w", err)
+	}
+
 	return nil
 }
 
@@ -100,7 +122,7 @@ func (mod *Module) Start() error {
 			case <-time.After(10 * time.Second):
 				count, err := mod.sqlStore.Cleanup(ctx, time.Now().Add(-network.DeleteConnsAfterEndedThreshold))
 				if err != nil {
-					log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
 				} else {
 					log.Infof("netquery: successfully removed %d old rows", count)
 				}
@@ -116,7 +138,7 @@ func (mod *Module) Start() error {
 			case <-time.After(5 * time.Second):
 				count, err := mod.sqlStore.CountRows(ctx)
 				if err != nil {
-					log.Errorf("netquery: failed to count number of rows in memory: %w", err)
+					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
 				} else {
 					log.Infof("netquery: currently holding %d rows in memory", count)
 				}
diff --git a/netquery/orm/decoder.go b/netquery/orm/decoder.go
index 5e0d1f7b..76359755 100644
--- a/netquery/orm/decoder.go
+++ b/netquery/orm/decoder.go
@@ -31,7 +31,7 @@ var (
 	// preconfigured timezone; UTC by default) or as INTEGER (the user can choose between
 	// unixepoch and unixnano-epoch where the nano variant is not offically supported by
 	// SQLITE).
-	sqliteTimeFormat = "2006-01-02 15:04:05"
+	SqliteTimeFormat = "2006-01-02 15:04:05"
 )
 
 type (
@@ -209,7 +209,7 @@ func DatetimeDecoder(loc *time.Location) DecodeFunc {
 		case sqlite.TypeText:
 			// stored ISO8601 but does not have any timezone information
 			// assigned so we always treat it as loc here.
-			t, err := time.ParseInLocation(sqliteTimeFormat, stmt.ColumnText(colIdx), loc)
+			t, err := time.ParseInLocation(SqliteTimeFormat, stmt.ColumnText(colIdx), loc)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse %q in %s: %w", stmt.ColumnText(colIdx), fieldDef.Name, err)
 			}
diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go
index fc4e772c..0b60f6ca 100644
--- a/netquery/orm/encoder.go
+++ b/netquery/orm/encoder.go
@@ -103,6 +103,13 @@ func encodeBasic() EncodeFunc {
 			kind = valType.Kind()
 
 			if val.IsNil() {
+				if !col.Nullable {
+					// we need to set the zero value here since the column
+					// is not marked as nullable
+					//return reflect.New(valType).Elem().Interface(), true, nil
+					panic("nil pointer for not-null field")
+				}
+
 				return nil, true, nil
 			}
 
@@ -133,7 +140,7 @@ func encodeBasic() EncodeFunc {
 }
 
 func DatetimeEncoder(loc *time.Location) EncodeFunc {
-	return func(colDev *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) {
+	return func(colDef *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) {
 		// if fieldType holds a pointer we need to dereference the value
 		ft := valType.String()
 		if valType.Kind() == reflect.Ptr {
@@ -142,44 +149,71 @@ func DatetimeEncoder(loc *time.Location) EncodeFunc {
 		}
 
 		// we only care about "time.Time" here
-		if ft != "time.Time" {
+		var t time.Time
+		if ft == "time.Time" {
+			// handle the zero time as a NULL.
+			if !val.IsValid() || val.IsZero() {
+				return nil, true, nil
+			}
+
+			var ok bool
+			valInterface := val.Interface()
+			t, ok = valInterface.(time.Time)
+			if !ok {
+				return nil, false, fmt.Errorf("cannot convert reflect value to time.Time")
+			}
+
+		} else if valType.Kind() == reflect.String && colDef.IsTime {
+			var err error
+			t, err = time.Parse(time.RFC3339, val.String())
+			if err != nil {
+				return nil, false, fmt.Errorf("failed to parse time as RFC3339: %w", err)
+			}
+
+		} else {
+			// we don't care ...
 			return nil, false, nil
 		}
 
-		// handle the zero time as a NULL.
-		if !val.IsValid() || val.IsZero() {
-			return nil, true, nil
-		}
-
-		valInterface := val.Interface()
-		t, ok := valInterface.(time.Time)
-		if !ok {
-			return nil, false, fmt.Errorf("cannot convert reflect value to time.Time")
-		}
-
-		switch colDev.Type {
+		switch colDef.Type {
 		case sqlite.TypeInteger:
-			if colDev.UnixNano {
+			if colDef.UnixNano {
 				return t.UnixNano(), true, nil
 			}
 			return t.Unix(), true, nil
+
 		case sqlite.TypeText:
-			str := t.In(loc).Format(sqliteTimeFormat)
+			str := t.In(loc).Format(SqliteTimeFormat)
 
 			return str, true, nil
 		}
 
-		return nil, false, fmt.Errorf("cannot store time.Time in %s", colDev.Type)
+		return nil, false, fmt.Errorf("cannot store time.Time in %s", colDef.Type)
 	}
 }
 
-func runEncodeHooks(colDev *ColumnDef, valType reflect.Type, val reflect.Value, hooks []EncodeFunc) (interface{}, bool, error) {
+func runEncodeHooks(colDef *ColumnDef, valType reflect.Type, val reflect.Value, hooks []EncodeFunc) (interface{}, bool, error) {
 	if valType == nil {
+		if !colDef.Nullable {
+			switch colDef.Type {
+			case sqlite.TypeBlob:
+				return []byte{}, true, nil
+			case sqlite.TypeFloat:
+				return 0.0, true, nil
+			case sqlite.TypeText:
+				return "", true, nil
+			case sqlite.TypeInteger:
+				return 0, true, nil
+			default:
+				return nil, false, fmt.Errorf("unsupported sqlite data type: %s", colDef.Type)
+			}
+		}
+
 		return nil, true, nil
 	}
 
 	for _, fn := range hooks {
-		res, end, err := fn(colDev, valType, val)
+		res, end, err := fn(colDef, valType, val)
 		if err != nil {
 			return res, false, err
 		}
diff --git a/netquery/orm/encoder_test.go b/netquery/orm/encoder_test.go
index 8b802a78..aff28580 100644
--- a/netquery/orm/encoder_test.go
+++ b/netquery/orm/encoder_test.go
@@ -89,7 +89,7 @@ func Test_EncodeAsMap(t *testing.T) {
 			},
 			map[string]interface{}{
 				"TinInt":    refTime.UnixNano(),
-				"TinString": refTime.Format(sqliteTimeFormat),
+				"TinString": refTime.Format(SqliteTimeFormat),
 			},
 		},
 		{
@@ -107,7 +107,7 @@ func Test_EncodeAsMap(t *testing.T) {
 			},
 			map[string]interface{}{
 				"TinInt":    refTime.UnixNano(),
-				"TinString": refTime.Format(sqliteTimeFormat),
+				"TinString": refTime.Format(SqliteTimeFormat),
 				"Tnil1":     nil,
 				"Tnil2":     nil,
 			},
@@ -143,7 +143,7 @@ func Test_EncodeValue(t *testing.T) {
 				Type:   sqlite.TypeText,
 			},
 			refTime,
-			refTime.Format(sqliteTimeFormat),
+			refTime.Format(SqliteTimeFormat),
 		},
 		{
 			"Special value time.Time as unix-epoch",
@@ -189,13 +189,14 @@ func Test_EncodeValue(t *testing.T) {
 				Type:   sqlite.TypeText,
 			},
 			&refTime,
-			refTime.Format(sqliteTimeFormat),
+			refTime.Format(SqliteTimeFormat),
 		},
 		{
 			"Special value untyped nil",
 			ColumnDef{
-				IsTime: true,
-				Type:   sqlite.TypeText,
+				Nullable: true,
+				IsTime:   true,
+				Type:     sqlite.TypeText,
 			},
 			nil,
 			nil,
@@ -209,12 +210,47 @@ func Test_EncodeValue(t *testing.T) {
 			(*time.Time)(nil),
 			nil,
 		},
+		{
+			"Time formated as string",
+			ColumnDef{
+				IsTime: true,
+				Type:   sqlite.TypeText,
+			},
+			refTime.In(time.Local).Format(time.RFC3339),
+			refTime.Format(SqliteTimeFormat),
+		},
+		{
+			"Nullable integer",
+			ColumnDef{
+				Type:     sqlite.TypeInteger,
+				Nullable: true,
+			},
+			nil,
+			nil,
+		},
+		{
+			"Not-Null integer",
+			ColumnDef{
+				Name: "test",
+				Type: sqlite.TypeInteger,
+			},
+			nil,
+			0,
+		},
+		{
+			"Not-Null string",
+			ColumnDef{
+				Type: sqlite.TypeText,
+			},
+			nil,
+			"",
+		},
 	}
 
 	for idx := range cases {
 		c := cases[idx]
 		t.Run(c.Desc, func(t *testing.T) {
-			// t.Parallel()
+			//t.Parallel()
 
 			res, err := EncodeValue(ctx, &c.Column, c.Input, DefaultEncodeConfig)
 			assert.NoError(t, err)
diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go
index 9289b06d..68783c3f 100644
--- a/netquery/orm/schema_builder.go
+++ b/netquery/orm/schema_builder.go
@@ -53,6 +53,15 @@ type (
 	}
 )
 
+func (ts TableSchema) GetColumnDef(name string) *ColumnDef {
+	for _, def := range ts.Columns {
+		if def.Name == name {
+			return &def
+		}
+	}
+	return nil
+}
+
 func (ts TableSchema) CreateStatement(ifNotExists bool) string {
 	sql := "CREATE TABLE"
 	if ifNotExists {
diff --git a/netquery/query.go b/netquery/query.go
new file mode 100644
index 00000000..2fdbf38f
--- /dev/null
+++ b/netquery/query.go
@@ -0,0 +1,464 @@
+package netquery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/safing/portmaster/netquery/orm"
+)
+
+type (
+	Query map[string][]Matcher
+
+	Matcher struct {
+		Equal    interface{}   `json:"$eq,omitempty"`
+		NotEqual interface{}   `json:"$ne,omitempty"`
+		In       []interface{} `json:"$in,omitempty"`
+		NotIn    []interface{} `json:"$notIn,omitempty"`
+		Like     string        `json:"$like,omitempty"`
+	}
+
+	Count struct {
+		As       string `json:"as"`
+		Field    string `json:"field"`
+		Distinct bool   `json:"distict"`
+	}
+
+	Select struct {
+		Field string `json:"field"`
+		Count *Count `json:"$count"`
+	}
+
+	Selects []Select
+
+	QueryRequestPayload struct {
+		Select  Selects   `json:"select"`
+		Query   Query     `json:"query"`
+		OrderBy []OrderBy `json:"orderBy"`
+		GroupBy []string  `json:"groupBy"`
+
+		selectedFields    []string
+		whitelistedFields []string
+	}
+
+	OrderBy struct {
+		Field string `json:"field"`
+		Desc  bool   `json:"desc"`
+	}
+
+	OrderBys []OrderBy
+)
+
+func (query *Query) UnmarshalJSON(blob []byte) error {
+	if *query == nil {
+		*query = make(Query)
+	}
+
+	var model map[string]json.RawMessage
+
+	if err := json.Unmarshal(blob, &model); err != nil {
+		return err
+	}
+
+	for columnName, rawColumnQuery := range model {
+		if len(rawColumnQuery) == 0 {
+			continue
+		}
+
+		switch rawColumnQuery[0] {
+		case '{':
+			m, err := parseMatcher(rawColumnQuery)
+			if err != nil {
+				return err
+			}
+
+			(*query)[columnName] = []Matcher{*m}
+
+		case '[':
+			var rawMatchers []json.RawMessage
+			if err := json.Unmarshal(rawColumnQuery, &rawMatchers); err != nil {
+				return err
+			}
+
+			(*query)[columnName] = make([]Matcher, len(rawMatchers))
+			for idx, val := range rawMatchers {
+				// this should not happen
+				if len(val) == 0 {
+					continue
+				}
+
+				// if val starts with a { we have a matcher definition
+				if val[0] == '{' {
+					m, err := parseMatcher(val)
+					if err != nil {
+						return err
+					}
+					(*query)[columnName][idx] = *m
+
+					continue
+				} else if val[0] == '[' {
+					return fmt.Errorf("invalid token [ in query for column %s", columnName)
+				}
+
+				// val is a dedicated JSON primitive and not an object or array
+				// so we treat that as an EQUAL condition.
+				var x interface{}
+				if err := json.Unmarshal(val, &x); err != nil {
+					return err
+				}
+
+				(*query)[columnName][idx] = Matcher{
+					Equal: x,
+				}
+			}
+
+		default:
+			// value is a JSON primitive and not an object or array
+			// so we treat that as an EQUAL condition.
+			var x interface{}
+			if err := json.Unmarshal(rawColumnQuery, &x); err != nil {
+				return err
+			}
+
+			(*query)[columnName] = []Matcher{
+				{Equal: x},
+			}
+		}
+	}
+
+	return nil
+}
+
+func parseMatcher(raw json.RawMessage) (*Matcher, error) {
+	var m Matcher
+	if err := json.Unmarshal(raw, &m); err != nil {
+		return nil, err
+	}
+
+	if err := m.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid query matcher: %s", err)
+	}
+	log.Printf("parsed matcher %s: %+v", string(raw), m)
+	return &m, nil
+
+}
+
+func (match Matcher) Validate() error {
+	found := 0
+
+	if match.Equal != nil {
+		found++
+	}
+
+	if match.NotEqual != nil {
+		found++
+	}
+
+	if match.In != nil {
+		found++
+	}
+
+	if match.NotIn != nil {
+		found++
+	}
+
+	if match.Like != "" {
+		found++
+	}
+
+	if found == 0 {
+		return fmt.Errorf("no conditions specified")
+	}
+
+	return nil
+}
+
+func (match Matcher) toSQLConditionClause(ctx context.Context, idx int, conjunction string, colDef orm.ColumnDef, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
+	var (
+		queryParts []string
+		params     = make(map[string]interface{})
+		errs       = new(multierror.Error)
+		key        = fmt.Sprintf("%s%d", colDef.Name, idx)
+	)
+
+	add := func(operator, suffix string, values ...interface{}) {
+		var placeholder []string
+
+		for idx, value := range values {
+			encodedValue, err := orm.EncodeValue(ctx, &colDef, value, encoderConfig)
+			if err != nil {
+				errs.Errors = append(errs.Errors,
+					fmt.Errorf("failed to encode %v for column %s: %w", value, colDef.Name, err),
+				)
+				return
+			}
+
+			uniqKey := fmt.Sprintf(":%s%s%d", key, suffix, idx)
+			placeholder = append(placeholder, uniqKey)
+			params[uniqKey] = encodedValue
+		}
+
+		if len(placeholder) == 1 {
+			queryParts = append(queryParts, fmt.Sprintf("%s %s %s", colDef.Name, operator, placeholder[0]))
+		} else {
+			queryParts = append(queryParts, fmt.Sprintf("%s %s ( %s )", colDef.Name, operator, strings.Join(placeholder, ", ")))
+		}
+	}
+
+	if match.Equal != nil {
+		add("=", "eq", match.Equal)
+	}
+
+	if match.NotEqual != nil {
+		add("!=", "ne", match.NotEqual)
+	}
+
+	if match.In != nil {
+		add("IN", "in", match.In...)
+	}
+
+	if match.NotIn != nil {
+		add("NOT IN", "notin", match.NotIn...)
+	}
+
+	if match.Like != "" {
+		add("LIKE", "like", match.Like)
+	}
+
+	if len(queryParts) == 0 {
+		// this is an empty matcher without a single condition.
+		// we convert that to a no-op TRUE value
+		return "( 1 = 1 )", nil, errs.ErrorOrNil()
+	}
+
+	if len(queryParts) == 1 {
+		return queryParts[0], params, errs.ErrorOrNil()
+	}
+
+	return "( " + strings.Join(queryParts, " "+conjunction+" ") + " )", params, errs.ErrorOrNil()
+}
+
+func (query Query) toSQLWhereClause(ctx context.Context, m *orm.TableSchema, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
+	if len(query) == 0 {
+		return "", nil, nil
+	}
+
+	// create a lookup map to validate column names
+	lm := make(map[string]orm.ColumnDef, len(m.Columns))
+	for _, col := range m.Columns {
+		lm[col.Name] = col
+	}
+
+	paramMap := make(map[string]interface{})
+	columnStmts := make([]string, 0, len(query))
+
+	// get all keys and sort them so we get a stable output
+	queryKeys := make([]string, 0, len(query))
+	for column := range query {
+		queryKeys = append(queryKeys, column)
+	}
+	sort.Strings(queryKeys)
+
+	// actually create the WHERE clause parts for each
+	// column in query.
+	errs := new(multierror.Error)
+	for _, column := range queryKeys {
+		values := query[column]
+		colDef, ok := lm[column]
+		if !ok {
+			errs.Errors = append(errs.Errors, fmt.Errorf("column %s is not allowed", column))
+
+			continue
+		}
+
+		queryParts := make([]string, len(values))
+		for idx, val := range values {
+			matcherQuery, params, err := val.toSQLConditionClause(ctx, idx, "AND", colDef, encoderConfig)
+			if err != nil {
+				errs.Errors = append(errs.Errors,
+					fmt.Errorf("invalid matcher at index %d for column %s: %w", idx, colDef.Name, err),
+				)
+
+				continue
+			}
+
+			// merge parameters up into the superior parameter map
+			for key, val := range params {
+				if _, ok := paramMap[key]; ok {
+					// is is soley a developer mistake when implementing a matcher so no forgiving ...
+					panic("sqlite parameter collision")
+				}
+
+				paramMap[key] = val
+			}
+
+			queryParts[idx] = matcherQuery
+		}
+
+		columnStmts = append(columnStmts,
+			fmt.Sprintf("( %s )", strings.Join(queryParts, " OR ")),
+		)
+	}
+
+	whereClause := strings.Join(columnStmts, " AND ")
+
+	return whereClause, paramMap, errs.ErrorOrNil()
+}
+
+func (sel *Selects) UnmarshalJSON(blob []byte) error {
+	if len(blob) == 0 {
+		return io.ErrUnexpectedEOF
+	}
+
+	// if we are looking at a slice directly decode into
+	// a []Select
+	if blob[0] == '[' {
+		var result []Select
+		if err := json.Unmarshal(blob, &result); err != nil {
+			return err
+		}
+
+		(*sel) = result
+
+		return nil
+	}
+
+	// if it's an object decode into a single select
+	if blob[0] == '{' {
+		var result Select
+		if err := json.Unmarshal(blob, &result); err != nil {
+			return err
+		}
+
+		*sel = []Select{result}
+
+		return nil
+	}
+
+	// otherwise this is just the field name
+	var field string
+	if err := json.Unmarshal(blob, &field); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (sel *Select) UnmarshalJSON(blob []byte) error {
+	if len(blob) == 0 {
+		return io.ErrUnexpectedEOF
+	}
+
+	// if we have an object at hand decode the select
+	// directly
+	if blob[0] == '{' {
+		var res struct {
+			Field string `json:"field"`
+			Count *Count `json:"$count"`
+		}
+
+		if err := json.Unmarshal(blob, &res); err != nil {
+			return err
+		}
+
+		sel.Count = res.Count
+		sel.Field = res.Field
+
+		if sel.Count != nil && sel.Count.As != "" {
+			if !charOnlyRegexp.MatchString(sel.Count.As) {
+				return fmt.Errorf("invalid characters in $count.as, value must match [a-zA-Z]+")
+			}
+		}
+
+		return nil
+	}
+
+	var x string
+	if err := json.Unmarshal(blob, &x); err != nil {
+		return err
+	}
+
+	sel.Field = x
+
+	return nil
+}
+
+func (orderBys *OrderBys) UnmarshalJSON(blob []byte) error {
+	if len(blob) == 0 {
+		return io.ErrUnexpectedEOF
+	}
+
+	if blob[0] == '[' {
+		var result []OrderBy
+		if err := json.Unmarshal(blob, &result); err != nil {
+			return err
+		}
+
+		*orderBys = result
+
+		return nil
+	}
+
+	if blob[0] == '{' {
+		var result OrderBy
+		if err := json.Unmarshal(blob, &result); err != nil {
+			return err
+		}
+
+		*orderBys = []OrderBy{result}
+
+		return nil
+	}
+
+	var field string
+	if err := json.Unmarshal(blob, &field); err != nil {
+		return err
+	}
+
+	*orderBys = []OrderBy{
+		{
+			Field: field,
+			Desc:  false,
+		},
+	}
+
+	return nil
+}
+
+func (orderBy *OrderBy) UnmarshalJSON(blob []byte) error {
+	if len(blob) == 0 {
+		return io.ErrUnexpectedEOF
+	}
+
+	if blob[0] == '{' {
+		var res struct {
+			Field string `json:"field"`
+			Desc  bool   `json:"desc"`
+		}
+
+		if err := json.Unmarshal(blob, &res); err != nil {
+			return err
+		}
+
+		orderBy.Desc = res.Desc
+		orderBy.Field = res.Field
+
+		return nil
+	}
+
+	var field string
+	if err := json.Unmarshal(blob, &field); err != nil {
+		return err
+	}
+
+	orderBy.Field = field
+	orderBy.Desc = false
+
+	return nil
+}
diff --git a/netquery/query_handler.go b/netquery/query_handler.go
new file mode 100644
index 00000000..a1e4353f
--- /dev/null
+++ b/netquery/query_handler.go
@@ -0,0 +1,293 @@
+package netquery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/netquery/orm"
+)
+
+var (
+	charOnlyRegexp = regexp.MustCompile("[a-zA-Z]+")
+)
+
+type (
+
+	// QueryHandler implements http.Handler and allows to perform SQL
+	// query and aggregate functions on Database.
+	QueryHandler struct {
+		IsDevMode func() bool
+		Database  *Database
+	}
+)
+
+func (qh *QueryHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	start := time.Now()
+	requestPayload, err := qh.parseRequest(req)
+	if err != nil {
+		http.Error(resp, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	queryParsed := time.Since(start)
+
+	query, paramMap, err := requestPayload.generateSQL(req.Context(), qh.Database.Schema)
+	if err != nil {
+		http.Error(resp, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	sqlQueryBuilt := time.Since(start)
+
+	// actually execute the query against the database and collect the result
+	var result []map[string]interface{}
+	if err := qh.Database.Execute(
+		req.Context(),
+		query,
+		orm.WithNamedArgs(paramMap),
+		orm.WithResult(&result),
+	); err != nil {
+		http.Error(resp, "Failed to execute query: "+err.Error(), http.StatusInternalServerError)
+
+		return
+	}
+	sqlQueryFinished := time.Since(start)
+
+	// send the HTTP status code
+	resp.WriteHeader(http.StatusOK)
+
+	// prepare the result encoder.
+	enc := json.NewEncoder(resp)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+
+	// prepare the result body that, in dev mode, contains
+	// some diagnostics data about the query
+	var resultBody map[string]interface{}
+	if qh.IsDevMode() {
+		resultBody = map[string]interface{}{
+			"sql_prep_stmt": query,
+			"sql_params":    paramMap,
+			"query":         requestPayload.Query,
+			"orderBy":       requestPayload.OrderBy,
+			"groupBy":       requestPayload.GroupBy,
+			"selects":       requestPayload.Select,
+			"times": map[string]interface{}{
+				"start_time":           start,
+				"query_parsed_after":   queryParsed.String(),
+				"query_built_after":    sqlQueryBuilt.String(),
+				"query_executed_after": sqlQueryFinished.String(),
+			},
+		}
+	} else {
+		resultBody = make(map[string]interface{})
+	}
+	resultBody["results"] = result
+
+	// and finally stream the response
+	if err := enc.Encode(resultBody); err != nil {
+		// we failed to encode the JSON body to resp so we likely either already sent a
+		// few bytes or the pipe was already closed. In either case, trying to send the
+		// error using http.Error() is non-sense. We just log it out here and that's all
+		// we can do.
+		log.Errorf("failed to encode JSON response: %s", err)
+
+		return
+	}
+}
+
+func (qh *QueryHandler) parseRequest(req *http.Request) (*QueryRequestPayload, error) {
+	var body io.Reader
+
+	switch req.Method {
+	case http.MethodPost, http.MethodPut:
+		body = req.Body
+	case http.MethodGet:
+		body = strings.NewReader(req.URL.Query().Get("q"))
+	default:
+		return nil, fmt.Errorf("invalid HTTP method")
+	}
+
+	var requestPayload QueryRequestPayload
+	blob, err := ioutil.ReadAll(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read body" + err.Error())
+	}
+
+	body = bytes.NewReader(blob)
+
+	dec := json.NewDecoder(body)
+	dec.DisallowUnknownFields()
+
+	if err := json.Unmarshal(blob, &requestPayload); err != nil && !errors.Is(err, io.EOF) {
+		return nil, fmt.Errorf("invalid query: %w", err)
+	}
+
+	return &requestPayload, nil
+}
+
+func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.TableSchema) (string, map[string]interface{}, error) {
+	if err := req.prepareSelectedFields(schema); err != nil {
+		return "", nil, fmt.Errorf("perparing selected fields: %w", err)
+	}
+
+	// build the SQL where clause from the payload query
+	whereClause, paramMap, err := req.Query.toSQLWhereClause(
+		ctx,
+		schema,
+		orm.DefaultEncodeConfig,
+	)
+	if err != nil {
+		return "", nil, fmt.Errorf("ganerating where clause: %w", err)
+	}
+
+	// build the actual SQL query statement
+	// FIXME(ppacher): add support for group-by and sort-by
+
+	groupByClause, err := req.generateGroupByClause(schema)
+	if err != nil {
+		return "", nil, fmt.Errorf("generating group-by clause: %w", err)
+	}
+
+	orderByClause, err := req.generateOrderByClause(schema)
+	if err != nil {
+		return "", nil, fmt.Errorf("generating order-by clause: %w", err)
+	}
+
+	selectClause := req.generateSelectClause()
+	query := `SELECT ` + selectClause + ` FROM connections`
+	if whereClause != "" {
+		query += " WHERE " + whereClause
+	}
+	query += " " + groupByClause + " " + orderByClause
+
+	return query, paramMap, nil
+}
+
+func (req *QueryRequestPayload) prepareSelectedFields(schema *orm.TableSchema) error {
+	for _, s := range req.Select {
+		var field string
+		if s.Count != nil {
+			field = s.Count.Field
+		} else {
+			field = s.Field
+		}
+
+		colName := "*"
+		if field != "*" || s.Count == nil {
+			var err error
+
+			colName, err = req.validateColumnName(schema, field)
+			if err != nil {
+				return err
+			}
+		}
+
+		if s.Count != nil {
+			var as = s.Count.As
+			if as == "" {
+				as = fmt.Sprintf("%s_count", colName)
+			}
+			var distinct = ""
+			if s.Count.Distinct {
+				distinct = "DISTINCT "
+			}
+			req.selectedFields = append(req.selectedFields, fmt.Sprintf("COUNT(%s%s) as %s", distinct, colName, as))
+			req.whitelistedFields = append(req.whitelistedFields, as)
+		} else {
+			req.selectedFields = append(req.selectedFields, colName)
+		}
+	}
+
+	return nil
+}
+
+func (req *QueryRequestPayload) generateGroupByClause(schema *orm.TableSchema) (string, error) {
+	if len(req.GroupBy) == 0 {
+		return "", nil
+	}
+
+	var groupBys = make([]string, len(req.GroupBy))
+
+	for idx, name := range req.GroupBy {
+		colName, err := req.validateColumnName(schema, name)
+		if err != nil {
+			return "", err
+		}
+
+		groupBys[idx] = colName
+	}
+
+	groupByClause := "GROUP BY " + strings.Join(groupBys, ", ")
+
+	// if there are no explicitly selected fields we default to the
+	// group-by columns as that's what's expected most of the time anyway...
+	if len(req.selectedFields) == 0 {
+		req.selectedFields = append(req.selectedFields, groupBys...)
+	}
+
+	return groupByClause, nil
+}
+
+func (req *QueryRequestPayload) generateSelectClause() string {
+	var selectClause = "*"
+	if len(req.selectedFields) > 0 {
+		selectClause = strings.Join(req.selectedFields, ", ")
+	}
+
+	return selectClause
+}
+
+func (req *QueryRequestPayload) generateOrderByClause(schema *orm.TableSchema) (string, error) {
+	var orderBys = make([]string, len(req.OrderBy))
+	for idx, sort := range req.OrderBy {
+		colName, err := req.validateColumnName(schema, sort.Field)
+		if err != nil {
+			return "", err
+		}
+
+		if sort.Desc {
+			orderBys[idx] = fmt.Sprintf("%s DESC", colName)
+		} else {
+			orderBys[idx] = fmt.Sprintf("%s ASC", colName)
+		}
+	}
+
+	return "ORDER BY " + strings.Join(orderBys, ", "), nil
+}
+
+func (req *QueryRequestPayload) validateColumnName(schema *orm.TableSchema, field string) (string, error) {
+	colDef := schema.GetColumnDef(field)
+	if colDef != nil {
+		return colDef.Name, nil
+	}
+
+	for _, selected := range req.whitelistedFields {
+		if field == selected {
+			return field, nil
+		}
+	}
+
+	for _, selected := range req.selectedFields {
+		if field == selected {
+			return field, nil
+		}
+	}
+
+	return "", fmt.Errorf("column name %s not allowed", field)
+}
+
+// compile time check
+var _ http.Handler = new(QueryHandler)
diff --git a/netquery/query_test.go b/netquery/query_test.go
new file mode 100644
index 00000000..f8e8b3e4
--- /dev/null
+++ b/netquery/query_test.go
@@ -0,0 +1,244 @@
+package netquery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/safing/portmaster/netquery/orm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_UnmarshalQuery(t *testing.T) {
+	var cases = []struct {
+		Name     string
+		Input    string
+		Expected Query
+		Error    error
+	}{
+		{
+			"Parse a simple query",
+			`{ "domain": ["example.com", "example.at"] }`,
+			Query{
+				"domain": []Matcher{
+					{
+						Equal: "example.com",
+					},
+					{
+						Equal: "example.at",
+					},
+				},
+			},
+			nil,
+		},
+		{
+			"Parse a more complex query",
+			`
+			{
+				"domain": [
+					{
+						"$in": [
+							"example.at",
+							"example.com"
+						]
+					},
+					{
+						"$like": "microsoft.%"
+					}
+				],
+				"path": [
+					"/bin/ping",
+					{
+						"$notin": [
+							"/sbin/ping",
+							"/usr/sbin/ping"
+						]
+					}
+				]
+			}
+			`,
+			Query{
+				"domain": []Matcher{
+					{
+						In: []interface{}{
+							"example.at",
+							"example.com",
+						},
+					},
+					{
+						Like: "microsoft.%",
+					},
+				},
+				"path": []Matcher{
+					{
+						Equal: "/bin/ping",
+					},
+					{
+						NotIn: []interface{}{
+							"/sbin/ping",
+							"/usr/sbin/ping",
+						},
+					},
+				},
+			},
+			nil,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			var q Query
+			err := json.Unmarshal([]byte(c.Input), &q)
+
+			if c.Error != nil {
+				if assert.Error(t, err) {
+					assert.Equal(t, c.Error.Error(), err.Error())
+				}
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, c.Expected, q)
+			}
+		})
+	}
+}
+
+func Test_QueryBuilder(t *testing.T) {
+	now := time.Now()
+
+	var cases = []struct {
+		N string
+		Q Query
+		R string
+		P map[string]interface{}
+		E error
+	}{
+		{
+			"No filter",
+			nil,
+			"",
+			nil,
+			nil,
+		},
+		{
+			"Simple, one-column filter",
+			Query{"domain": []Matcher{
+				{
+					Equal: "example.com",
+				},
+				{
+					Equal: "example.at",
+				},
+			}},
+			"( domain = :domain0eq0 OR domain = :domain1eq0 )",
+			map[string]interface{}{
+				":domain0eq0": "example.com",
+				":domain1eq0": "example.at",
+			},
+			nil,
+		},
+		{
+			"Two column filter",
+			Query{
+				"domain": []Matcher{
+					{
+						Equal: "example.com",
+					},
+				},
+				"path": []Matcher{
+					{
+						Equal: "/bin/curl",
+					},
+					{
+						Equal: "/bin/ping",
+					},
+				},
+			},
+			"( domain = :domain0eq0 ) AND ( path = :path0eq0 OR path = :path1eq0 )",
+			map[string]interface{}{
+				":domain0eq0": "example.com",
+				":path0eq0":   "/bin/curl",
+				":path1eq0":   "/bin/ping",
+			},
+			nil,
+		},
+		{
+			"Time based filter",
+			Query{
+				"started": []Matcher{
+					{
+						Equal: now.Format(time.RFC3339),
+					},
+				},
+			},
+			"( started = :started0eq0 )",
+			map[string]interface{}{
+				":started0eq0": now.In(time.UTC).Format(orm.SqliteTimeFormat),
+			},
+			nil,
+		},
+		{
+			"Invalid column access",
+			Query{
+				"forbiddenField": []Matcher{{}},
+			},
+			"",
+			nil,
+			fmt.Errorf("1 error occurred:\n\t* column forbiddenField is not allowed\n\n"),
+		},
+		{
+			"Complex example",
+			Query{
+				"domain": []Matcher{
+					{
+						In: []interface{}{"example.at", "example.com"},
+					},
+					{
+						Like: "microsoft.%",
+					},
+				},
+				"path": []Matcher{
+					{
+						NotIn: []interface{}{
+							"/bin/ping",
+							"/sbin/ping",
+							"/usr/bin/ping",
+						},
+					},
+				},
+			},
+			"( domain IN ( :domain0in0, :domain0in1 ) OR domain LIKE :domain1like0 ) AND ( path NOT IN ( :path0notin0, :path0notin1, :path0notin2 ) )",
+			map[string]interface{}{
+				":domain0in0":   "example.at",
+				":domain0in1":   "example.com",
+				":domain1like0": "microsoft.%",
+				":path0notin0":  "/bin/ping",
+				":path0notin1":  "/sbin/ping",
+				":path0notin2":  "/usr/bin/ping",
+			},
+			nil,
+		},
+	}
+
+	tbl, err := orm.GenerateTableSchema("connections", Conn{})
+	require.NoError(t, err)
+
+	for idx, c := range cases {
+		t.Run(c.N, func(t *testing.T) {
+			//t.Parallel()
+			str, params, err := c.Q.toSQLWhereClause(context.TODO(), tbl, orm.DefaultEncodeConfig)
+
+			if c.E != nil {
+				if assert.Error(t, err) {
+					assert.Equal(t, c.E.Error(), err.Error(), "test case %d", idx)
+				}
+			} else {
+				assert.NoError(t, err, "test case %d", idx)
+				assert.Equal(t, c.P, params, "test case %d", idx)
+				assert.Equal(t, c.R, str, "test case %d", idx)
+			}
+		})
+	}
+}

From 0d2ec9df756c37582d2eb69881a35b9d8a7f07b1 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 26 Apr 2022 15:03:04 +0200
Subject: [PATCH 59/86] wip

---
 netquery/orm/encoder.go | 3 +--
 netquery/query.go       | 8 ++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go
index 0b60f6ca..0bcea756 100644
--- a/netquery/orm/encoder.go
+++ b/netquery/orm/encoder.go
@@ -106,8 +106,7 @@ func encodeBasic() EncodeFunc {
 				if !col.Nullable {
 					// we need to set the zero value here since the column
 					// is not marked as nullable
-					//return reflect.New(valType).Elem().Interface(), true, nil
-					panic("nil pointer for not-null field")
+					return reflect.New(valType).Elem().Interface(), true, nil
 				}
 
 				return nil, true, nil
diff --git a/netquery/query.go b/netquery/query.go
index 2fdbf38f..6febf0b6 100644
--- a/netquery/query.go
+++ b/netquery/query.go
@@ -38,10 +38,10 @@ type (
 	Selects []Select
 
 	QueryRequestPayload struct {
-		Select  Selects   `json:"select"`
-		Query   Query     `json:"query"`
-		OrderBy []OrderBy `json:"orderBy"`
-		GroupBy []string  `json:"groupBy"`
+		Select  Selects  `json:"select"`
+		Query   Query    `json:"query"`
+		OrderBy OrderBys `json:"orderBy"`
+		GroupBy []string `json:"groupBy"`
 
 		selectedFields    []string
 		whitelistedFields []string

From 25aceaf103743e835cf83135ffa12ccead0d7da3 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 3 May 2022 16:11:12 +0200
Subject: [PATCH 60/86] Add query and chart support with multiple fixes to ORM
 package

---
 netquery/chart_handler.go      | 118 ++++++++++++++++++++++++++
 netquery/database.go           |  61 +++++++-------
 netquery/manager.go            |  53 ++++++++----
 netquery/module_api.go         |  24 +++++-
 netquery/orm/decoder.go        | 148 +++++++++++++++++++++++++--------
 netquery/orm/decoder_test.go   | 110 ++++++++++++++++++++++--
 netquery/orm/query_runner.go   |   9 +-
 netquery/orm/schema_builder.go |   2 +
 netquery/query.go              |  57 +++++++++----
 netquery/query_handler.go      |  68 ++++++++++++---
 netquery/query_test.go         |   2 +-
 11 files changed, 535 insertions(+), 117 deletions(-)
 create mode 100644 netquery/chart_handler.go

diff --git a/netquery/chart_handler.go b/netquery/chart_handler.go
new file mode 100644
index 00000000..22790fb5
--- /dev/null
+++ b/netquery/chart_handler.go
@@ -0,0 +1,118 @@
+package netquery
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/safing/portmaster/netquery/orm"
+)
+
+type ChartHandler struct {
+	Database *Database
+}
+
+func (ch *ChartHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	requestPayload, err := ch.parseRequest(req)
+	if err != nil {
+		http.Error(resp, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	query, paramMap, err := requestPayload.generateSQL(req.Context(), ch.Database.Schema)
+	if err != nil {
+		http.Error(resp, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	// actually execute the query against the database and collect the result
+	var result []map[string]interface{}
+	if err := ch.Database.Execute(
+		req.Context(),
+		query,
+		orm.WithNamedArgs(paramMap),
+		orm.WithResult(&result),
+		orm.WithSchema(*ch.Database.Schema),
+	); err != nil {
+		http.Error(resp, "Failed to execute query: "+err.Error(), http.StatusInternalServerError)
+
+		return
+	}
+
+	// send the HTTP status code
+	resp.WriteHeader(http.StatusOK)
+
+	// prepare the result encoder.
+	enc := json.NewEncoder(resp)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+
+	enc.Encode(map[string]interface{}{
+		"results": result,
+		"query":   query,
+		"params":  paramMap,
+	})
+}
+
+func (ch *ChartHandler) parseRequest(req *http.Request) (*QueryActiveConnectionChartPayload, error) {
+	var body io.Reader
+
+	switch req.Method {
+	case http.MethodPost, http.MethodPut:
+		body = req.Body
+	case http.MethodGet:
+		body = strings.NewReader(req.URL.Query().Get("q"))
+	default:
+		return nil, fmt.Errorf("invalid HTTP method")
+	}
+
+	var requestPayload QueryActiveConnectionChartPayload
+	blob, err := ioutil.ReadAll(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read body" + err.Error())
+	}
+
+	body = bytes.NewReader(blob)
+
+	dec := json.NewDecoder(body)
+	dec.DisallowUnknownFields()
+
+	if err := json.Unmarshal(blob, &requestPayload); err != nil && !errors.Is(err, io.EOF) {
+		return nil, fmt.Errorf("invalid query: %w", err)
+	}
+
+	return &requestPayload, nil
+}
+
+func (req *QueryActiveConnectionChartPayload) generateSQL(ctx context.Context, schema *orm.TableSchema) (string, map[string]interface{}, error) {
+	template := `
+WITH RECURSIVE epoch(x) AS (
+	SELECT strftime('%%s')-600
+	UNION ALL
+		SELECT x+1 FROM epoch WHERE x+1 < strftime('%%s')+0
+)
+SELECT x as timestamp, COUNT(*) AS value FROM epoch
+	JOIN connections
+	ON strftime('%%s', connections.started)+0 <= timestamp+0 AND (connections.ended IS NULL OR strftime('%%s', connections.ended)+0 > timestamp+0)
+		%s
+	GROUP BY round(timestamp/10, 0)*10;`
+
+	clause, params, err := req.Query.toSQLWhereClause(ctx, "", schema, orm.DefaultEncodeConfig)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if clause == "" {
+		return fmt.Sprintf(template, ""), map[string]interface{}{}, nil
+	}
+
+	return fmt.Sprintf(template, "WHERE ( "+clause+")"), params, nil
+}
diff --git a/netquery/database.go b/netquery/database.go
index 25f9e571..4c53ebdd 100644
--- a/netquery/database.go
+++ b/netquery/database.go
@@ -63,32 +63,35 @@ type (
 		// time. We cannot just use the network.Connection.ID because it is only unique
 		// as long as the connection is still active and might be, although unlikely,
 		// reused afterwards.
-		ID         string            `sqlite:"id,primary"`
-		ProfileID  string            `sqlite:"profile"`
-		Path       string            `sqlite:"path"`
-		Type       string            `sqlite:"type,varchar(8)"`
-		External   bool              `sqlite:"external"`
-		IPVersion  packet.IPVersion  `sqlite:"ip_version"`
-		IPProtocol packet.IPProtocol `sqlite:"ip_protocol"`
-		LocalIP    string            `sqlite:"local_ip"`
-		LocalPort  uint16            `sqlite:"local_port"`
-		RemoteIP   string            `sqlite:"remote_ip"`
-		RemotePort uint16            `sqlite:"remote_port"`
-		Domain     string            `sqlite:"domain"`
-		Country    string            `sqlite:"country,varchar(2)"`
-		ASN        uint              `sqlite:"asn"`
-		ASOwner    string            `sqlite:"as_owner"`
-		Latitude   float64           `sqlite:"latitude"`
-		Longitude  float64           `sqlite:"longitude"`
-		Scope      netutils.IPScope  `sqlite:"scope"`
-		Verdict    network.Verdict   `sqlite:"verdict"`
-		Started    time.Time         `sqlite:"started,text,time"`
-		Ended      *time.Time        `sqlite:"ended,text,time"`
-		Tunneled   bool              `sqlite:"tunneled"`
-		Encrypted  bool              `sqlite:"encrypted"`
-		Internal   bool              `sqlite:"internal"`
-		Inbound    bool              `sqlite:"inbound"`
-		ExtraData  json.RawMessage   `sqlite:"extra_data"`
+		ID              string            `sqlite:"id,primary"`
+		ProfileID       string            `sqlite:"profile"`
+		ProfileSource   string            `sqlite:"profileSource"`
+		Path            string            `sqlite:"path"`
+		Type            string            `sqlite:"type,varchar(8)"`
+		External        bool              `sqlite:"external"`
+		IPVersion       packet.IPVersion  `sqlite:"ip_version"`
+		IPProtocol      packet.IPProtocol `sqlite:"ip_protocol"`
+		LocalIP         string            `sqlite:"local_ip"`
+		LocalPort       uint16            `sqlite:"local_port"`
+		RemoteIP        string            `sqlite:"remote_ip"`
+		RemotePort      uint16            `sqlite:"remote_port"`
+		Domain          string            `sqlite:"domain"`
+		Country         string            `sqlite:"country,varchar(2)"`
+		ASN             uint              `sqlite:"asn"`
+		ASOwner         string            `sqlite:"as_owner"`
+		Latitude        float64           `sqlite:"latitude"`
+		Longitude       float64           `sqlite:"longitude"`
+		Scope           netutils.IPScope  `sqlite:"scope"`
+		Verdict         network.Verdict   `sqlite:"verdict"`
+		Started         time.Time         `sqlite:"started,text,time"`
+		Ended           *time.Time        `sqlite:"ended,text,time"`
+		Tunneled        bool              `sqlite:"tunneled"`
+		Encrypted       bool              `sqlite:"encrypted"`
+		Internal        bool              `sqlite:"internal"`
+		Direction       string            `sqlite:"direction"`
+		ExtraData       json.RawMessage   `sqlite:"extra_data"`
+		Allowed         *bool             `sqlite:"allowed"`
+		ProfileRevision int               `sqlite:"profile_revision"`
 	}
 )
 
@@ -190,11 +193,11 @@ func (db *Database) CountRows(ctx context.Context) (int, error) {
 // probably not worth the cylces...
 func (db *Database) Cleanup(ctx context.Context, threshold time.Time) (int, error) {
 	where := `WHERE ended IS NOT NULL
-			AND datetime(ended) < :threshold`
+			AND datetime(ended) < datetime(:threshold)`
 	sql := "DELETE FROM connections " + where + ";"
 
 	args := orm.WithNamedArgs(map[string]interface{}{
-		":threshold": threshold,
+		":threshold": threshold.UTC().Format(orm.SqliteTimeFormat),
 	})
 
 	var result []struct {
@@ -232,7 +235,7 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error {
 	if err := sqlitex.ExecuteTransient(db.conn, "SELECT * FROM connections", &sqlitex.ExecOptions{
 		ResultFunc: func(stmt *sqlite.Stmt) error {
 			var c Conn
-			if err := orm.DecodeStmt(ctx, stmt, &c, orm.DefaultDecodeConfig); err != nil {
+			if err := orm.DecodeStmt(ctx, db.Schema, stmt, &c, orm.DefaultDecodeConfig); err != nil {
 				return err
 			}
 
diff --git a/netquery/manager.go b/netquery/manager.go
index 16a78021..559ccdfe 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -155,23 +155,42 @@ func (mng *Manager) pushConnUpdate(ctx context.Context, meta record.Meta, conn C
 func convertConnection(conn *network.Connection) (*Conn, error) {
 	conn.Lock()
 	defer conn.Unlock()
+	direction := "outbound"
+	if conn.Inbound {
+		direction = "inbound"
+	}
 
 	c := Conn{
-		ID:         genConnID(conn),
-		External:   conn.External,
-		IPVersion:  conn.IPVersion,
-		IPProtocol: conn.IPProtocol,
-		LocalIP:    conn.LocalIP.String(),
-		LocalPort:  conn.LocalPort,
-		Verdict:    conn.Verdict,
-		Started:    time.Unix(conn.Started, 0),
-		Tunneled:   conn.Tunneled,
-		Encrypted:  conn.Encrypted,
-		Internal:   conn.Internal,
-		Inbound:    conn.Inbound,
-		Type:       ConnectionTypeToString[conn.Type],
-		ProfileID:  conn.ProcessContext.ProfileName,
-		Path:       conn.ProcessContext.BinaryPath,
+		ID:              genConnID(conn),
+		External:        conn.External,
+		IPVersion:       conn.IPVersion,
+		IPProtocol:      conn.IPProtocol,
+		LocalIP:         conn.LocalIP.String(),
+		LocalPort:       conn.LocalPort,
+		Verdict:         conn.Verdict,
+		Started:         time.Unix(conn.Started, 0),
+		Tunneled:        conn.Tunneled,
+		Encrypted:       conn.Encrypted,
+		Internal:        conn.Internal,
+		Direction:       direction,
+		Type:            ConnectionTypeToString[conn.Type],
+		ProfileID:       conn.ProcessContext.Profile,
+		ProfileSource:   conn.ProcessContext.Source,
+		Path:            conn.ProcessContext.BinaryPath,
+		ProfileRevision: int(conn.ProfileRevisionCounter),
+	}
+
+	switch conn.Type {
+	case network.DNSRequest:
+		c.Type = "dns"
+	case network.IPConnection:
+		c.Type = "ip"
+	}
+
+	switch conn.Verdict {
+	case network.VerdictAccept, network.VerdictRerouteToNameserver, network.VerdictRerouteToTunnel:
+		accepted := true
+		c.Allowed = &accepted
 	}
 
 	if conn.Ended > 0 {
@@ -181,6 +200,10 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 
 	extraData := map[string]interface{}{}
 
+	if conn.TunnelContext != nil {
+		extraData["tunnel"] = conn.TunnelContext
+	}
+
 	if conn.Entity != nil {
 		extraData["cname"] = conn.Entity.CNAME
 		extraData["blockedByLists"] = conn.Entity.BlockedByLists
diff --git a/netquery/module_api.go b/netquery/module_api.go
index a344ec3b..dd15189e 100644
--- a/netquery/module_api.go
+++ b/netquery/module_api.go
@@ -63,6 +63,10 @@ func (m *Module) Prepare() error {
 		IsDevMode: config.Concurrent.GetAsBool(config.CfgDevModeKey, false),
 	}
 
+	chartHandler := &ChartHandler{
+		Database: m.sqlStore,
+	}
+
 	// FIXME(ppacher): use appropriate permissions for this
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:        "netquery/query",
@@ -77,6 +81,19 @@ func (m *Module) Prepare() error {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
 
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:        "netquery/charts/connection-active",
+		MimeType:    "application/json",
+		Read:        api.PermitAnyone,
+		Write:       api.PermitAnyone,
+		BelongsTo:   m.Module,
+		HandlerFunc: chartHandler.ServeHTTP,
+		Name:        "Query In-Memory Database",
+		Description: "Query the in-memory sqlite database",
+	}); err != nil {
+		return fmt.Errorf("failed to register API endpoint: %w", err)
+	}
+
 	return nil
 }
 
@@ -120,11 +137,12 @@ func (mod *Module) Start() error {
 			case <-ctx.Done():
 				return nil
 			case <-time.After(10 * time.Second):
-				count, err := mod.sqlStore.Cleanup(ctx, time.Now().Add(-network.DeleteConnsAfterEndedThreshold))
+				threshold := time.Now().Add(-network.DeleteConnsAfterEndedThreshold)
+				count, err := mod.sqlStore.Cleanup(ctx, threshold)
 				if err != nil {
 					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
 				} else {
-					log.Infof("netquery: successfully removed %d old rows", count)
+					log.Infof("netquery: successfully removed %d old rows that ended before %s", count, threshold)
 				}
 			}
 		}
@@ -135,7 +153,7 @@ func (mod *Module) Start() error {
 			select {
 			case <-ctx.Done():
 				return nil
-			case <-time.After(5 * time.Second):
+			case <-time.After(1 * time.Second):
 				count, err := mod.sqlStore.CountRows(ctx)
 				if err != nil {
 					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
diff --git a/netquery/orm/decoder.go b/netquery/orm/decoder.go
index 76359755..6cc16f97 100644
--- a/netquery/orm/decoder.go
+++ b/netquery/orm/decoder.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"reflect"
 	"strings"
 	"time"
@@ -51,7 +52,7 @@ type (
 	}
 
 	// DecodeFunc is called for each non-basic type during decoding.
-	DecodeFunc func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error)
+	DecodeFunc func(colIdx int, colDef *ColumnDef, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, bool, error)
 
 	DecodeConfig struct {
 		DecodeHooks []DecodeFunc
@@ -63,7 +64,7 @@ type (
 // be specified to provide support for special types.
 // See DatetimeDecoder() for an example of a DecodeHook that handles graceful time.Time conversion.
 //
-func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeConfig) error {
+func DecodeStmt(ctx context.Context, schema *TableSchema, stmt Stmt, result interface{}, cfg DecodeConfig) error {
 	// make sure we got something to decode into ...
 	if result == nil {
 		return fmt.Errorf("%w, got %T", errStructPointerExpected, result)
@@ -71,7 +72,7 @@ func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeCo
 
 	// fast path for decoding into a map
 	if mp, ok := result.(*map[string]interface{}); ok {
-		return decodeIntoMap(ctx, stmt, mp)
+		return decodeIntoMap(ctx, schema, stmt, mp, cfg)
 	}
 
 	// make sure we got a pointer in result
@@ -147,10 +148,13 @@ func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeCo
 			value = storage.Elem()
 		}
 
+		colDef := schema.GetColumnDef(colName)
+
 		// execute all decode hooks but make sure we use decodeBasic() as the
 		// last one.
 		columnValue, err := runDecodeHooks(
 			i,
+			colDef,
 			stmt,
 			fieldType,
 			value,
@@ -188,10 +192,19 @@ func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeCo
 // FIXME(ppacher): update comment about loc parameter and TEXT storage class parsing
 //
 func DatetimeDecoder(loc *time.Location) DecodeFunc {
-	return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) {
+	return func(colIdx int, colDef *ColumnDef, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, bool, error) {
+		// if we have the column definition available we
+		// use the target go type from there.
+		outType := outval.Type()
+
+		if colDef != nil {
+			outType = colDef.GoType
+		}
+
 		// we only care about "time.Time" here
-		if outval.Type().String() != "time.Time" {
-			return nil, nil
+		if outType.String() != "time.Time" || (colDef != nil && !colDef.IsTime) {
+			log.Printf("not decoding %s %v", outType, colDef)
+			return nil, false, nil
 		}
 
 		switch stmt.ColumnType(colIdx) {
@@ -201,39 +214,61 @@ func DatetimeDecoder(loc *time.Location) DecodeFunc {
 			// TODO(ppacher): actually split the tag value at "," and search
 			// the slice for "unixnano"
 			if strings.Contains(fieldDef.Tag.Get("sqlite"), ",unixnano") {
-				return time.Unix(0, int64(stmt.ColumnInt(colIdx))), nil
+				return time.Unix(0, int64(stmt.ColumnInt(colIdx))), true, nil
 			}
 
-			return time.Unix(int64(stmt.ColumnInt(colIdx)), 0), nil
+			return time.Unix(int64(stmt.ColumnInt(colIdx)), 0), true, nil
 
 		case sqlite.TypeText:
 			// stored ISO8601 but does not have any timezone information
 			// assigned so we always treat it as loc here.
 			t, err := time.ParseInLocation(SqliteTimeFormat, stmt.ColumnText(colIdx), loc)
 			if err != nil {
-				return nil, fmt.Errorf("failed to parse %q in %s: %w", stmt.ColumnText(colIdx), fieldDef.Name, err)
+				return nil, false, fmt.Errorf("failed to parse %q in %s: %w", stmt.ColumnText(colIdx), fieldDef.Name, err)
 			}
 
-			return t, nil
+			return t, true, nil
 
 		case sqlite.TypeFloat:
 			// stored as Julian day numbers
-			return nil, fmt.Errorf("REAL storage type not support for time.Time")
+			return nil, false, fmt.Errorf("REAL storage type not support for time.Time")
+
+		case sqlite.TypeNull:
+			return nil, true, nil
 
 		default:
-			return nil, fmt.Errorf("unsupported storage type for time.Time: %s", outval.Type())
+			return nil, false, fmt.Errorf("unsupported storage type for time.Time: %s", stmt.ColumnType(colIdx))
 		}
 	}
 }
 
-func decodeIntoMap(ctx context.Context, stmt Stmt, mp *map[string]interface{}) error {
+func decodeIntoMap(ctx context.Context, schema *TableSchema, stmt Stmt, mp *map[string]interface{}, cfg DecodeConfig) error {
 	if *mp == nil {
 		*mp = make(map[string]interface{})
 	}
 
 	for i := 0; i < stmt.ColumnCount(); i++ {
 		var x interface{}
-		val, err := decodeBasic()(i, stmt, reflect.StructField{}, reflect.ValueOf(&x).Elem())
+
+		colDef := schema.GetColumnDef(stmt.ColumnName(i))
+
+		outVal := reflect.ValueOf(&x).Elem()
+		fieldType := reflect.StructField{}
+		if colDef != nil {
+			outVal = reflect.New(colDef.GoType).Elem()
+			fieldType = reflect.StructField{
+				Type: colDef.GoType,
+			}
+		}
+
+		val, err := runDecodeHooks(
+			i,
+			colDef,
+			stmt,
+			fieldType,
+			outVal,
+			append(cfg.DecodeHooks, decodeBasic()),
+		)
 		if err != nil {
 			return fmt.Errorf("failed to decode column %s: %w", stmt.ColumnName(i), err)
 		}
@@ -245,56 +280,99 @@ func decodeIntoMap(ctx context.Context, stmt Stmt, mp *map[string]interface{}) e
 }
 
 func decodeBasic() DecodeFunc {
-	return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) {
+	return func(colIdx int, colDef *ColumnDef, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (result interface{}, handled bool, err error) {
 		valueKind := getKind(outval)
 		colType := stmt.ColumnType(colIdx)
 		colName := stmt.ColumnName(colIdx)
 
 		errInvalidType := fmt.Errorf("%w %s for column %s with field type %s", errUnexpectedColumnType, colType.String(), colName, outval.Type())
 
+		// if we have the column definition available we
+		// use the target go type from there.
+		if colDef != nil {
+			valueKind = normalizeKind(colDef.GoType.Kind())
+
+			// if we have a column defintion we try to convert the value to
+			// the actual Go-type that was used in the model.
+			// this is useful, for example, to ensure a []byte{} is always decoded into json.RawMessage
+			// or that type aliases like (type myInt int) are decoded into myInt instead of int
+			defer func() {
+				if handled {
+					t := reflect.New(colDef.GoType).Elem()
+
+					if result == nil || reflect.ValueOf(result).IsZero() {
+						return
+					}
+
+					if reflect.ValueOf(result).Type().ConvertibleTo(colDef.GoType) {
+						result = reflect.ValueOf(result).Convert(colDef.GoType).Interface()
+					}
+					t.Set(reflect.ValueOf(result))
+
+					result = t.Interface()
+				}
+			}()
+		}
+
+		log.Printf("decoding %s into kind %s", colName, valueKind)
+
+		if colType == sqlite.TypeNull {
+			if colDef != nil && colDef.Nullable {
+				return nil, true, nil
+			}
+
+			if colDef != nil && !colDef.Nullable {
+				return reflect.New(colDef.GoType).Elem().Interface(), true, nil
+			}
+
+			if outval.Kind() == reflect.Ptr {
+				return nil, true, nil
+			}
+		}
+
 		switch valueKind {
 		case reflect.String:
 			if colType != sqlite.TypeText {
-				return nil, errInvalidType
+				return nil, false, errInvalidType
 			}
-			return stmt.ColumnText(colIdx), nil
+			return stmt.ColumnText(colIdx), true, nil
 
 		case reflect.Bool:
 			// sqlite does not have a BOOL type, it rather stores a 1/0 in a column
 			// with INTEGER affinity.
 			if colType != sqlite.TypeInteger {
-				return nil, errInvalidType
+				return nil, false, errInvalidType
 			}
-			return stmt.ColumnBool(colIdx), nil
+			return stmt.ColumnBool(colIdx), true, nil
 
 		case reflect.Float64:
 			if colType != sqlite.TypeFloat {
-				return nil, errInvalidType
+				return nil, false, errInvalidType
 			}
-			return stmt.ColumnFloat(colIdx), nil
+			return stmt.ColumnFloat(colIdx), true, nil
 
 		case reflect.Int, reflect.Uint: // getKind() normalizes all ints to reflect.Int/Uint because sqlite doesn't really care ...
 			if colType != sqlite.TypeInteger {
-				return nil, errInvalidType
+				return nil, false, errInvalidType
 			}
 
-			return stmt.ColumnInt(colIdx), nil
+			return stmt.ColumnInt(colIdx), true, nil
 
 		case reflect.Slice:
 			if outval.Type().Elem().Kind() != reflect.Uint8 {
-				return nil, fmt.Errorf("slices other than []byte for BLOB are not supported")
+				return nil, false, fmt.Errorf("slices other than []byte for BLOB are not supported")
 			}
 
 			if colType != sqlite.TypeBlob {
-				return nil, errInvalidType
+				return nil, false, errInvalidType
 			}
 
 			columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx))
 			if err != nil {
-				return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
+				return nil, false, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
 			}
 
-			return columnValue, nil
+			return columnValue, true, nil
 
 		case reflect.Interface:
 			var (
@@ -306,7 +384,7 @@ func decodeBasic() DecodeFunc {
 				t = reflect.TypeOf([]byte{})
 				columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx))
 				if err != nil {
-					return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
+					return nil, false, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err)
 				}
 				x = columnValue
 
@@ -327,20 +405,20 @@ func decodeBasic() DecodeFunc {
 				x = nil
 
 			default:
-				return nil, fmt.Errorf("unsupported column type %s", colType)
+				return nil, false, fmt.Errorf("unsupported column type %s", colType)
 			}
 
 			if t == nil {
-				return nil, nil
+				return nil, true, nil
 			}
 
 			target := reflect.New(t).Elem()
 			target.Set(reflect.ValueOf(x))
 
-			return target.Interface(), nil
+			return target.Interface(), true, nil
 
 		default:
-			return nil, fmt.Errorf("cannot decode into %s", valueKind)
+			return nil, false, fmt.Errorf("cannot decode into %s", valueKind)
 		}
 	}
 }
@@ -362,14 +440,14 @@ func sqlColumnName(fieldType reflect.StructField) string {
 // runDecodeHooks tries to decode the column value of stmt at index colIdx into outval by running all decode hooks.
 // The first hook that returns a non-nil interface wins, other hooks will not be executed. If an error is
 // returned by a decode hook runDecodeHooks stops the error is returned to the caller.
-func runDecodeHooks(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value, hooks []DecodeFunc) (interface{}, error) {
+func runDecodeHooks(colIdx int, colDef *ColumnDef, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value, hooks []DecodeFunc) (interface{}, error) {
 	for _, fn := range hooks {
-		res, err := fn(colIdx, stmt, fieldDef, outval)
+		res, end, err := fn(colIdx, colDef, stmt, fieldDef, outval)
 		if err != nil {
 			return res, err
 		}
 
-		if res != nil {
+		if end {
 			return res, nil
 		}
 	}
diff --git a/netquery/orm/decoder_test.go b/netquery/orm/decoder_test.go
index c6c836c0..5abd324e 100644
--- a/netquery/orm/decoder_test.go
+++ b/netquery/orm/decoder_test.go
@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"log"
+	"reflect"
 	"testing"
 	"time"
 
@@ -82,6 +84,8 @@ func (ett *exampleTimeTypes) Equal(other interface{}) bool {
 	return ett.T.Equal(oett.T) && (ett.TP != nil && oett.TP != nil && ett.TP.Equal(*oett.TP)) || (ett.TP == nil && oett.TP == nil)
 }
 
+type myInt int
+
 type exampleTimeNano struct {
 	T time.Time `sqlite:",unixnano"`
 }
@@ -100,10 +104,11 @@ func Test_Decoder(t *testing.T) {
 	refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC)
 
 	cases := []struct {
-		Desc     string
-		Stmt     testStmt
-		Result   interface{}
-		Expected interface{}
+		Desc      string
+		Stmt      testStmt
+		ColumnDef []ColumnDef
+		Result    interface{}
+		Expected  interface{}
 	}{
 		{
 			"Decoding into nil is not allowed",
@@ -114,6 +119,7 @@ func Test_Decoder(t *testing.T) {
 			},
 			nil,
 			nil,
+			nil,
 		},
 		{
 			"Decoding into basic types",
@@ -132,6 +138,7 @@ func Test_Decoder(t *testing.T) {
 					true,
 				},
 			},
+			nil,
 			&exampleFieldTypes{},
 			&exampleFieldTypes{
 				S: "string value",
@@ -157,6 +164,7 @@ func Test_Decoder(t *testing.T) {
 					1.2,
 				},
 			},
+			nil,
 			&exampleFieldTypes{},
 			&exampleFieldTypes{
 				S: "string value",
@@ -178,6 +186,7 @@ func Test_Decoder(t *testing.T) {
 					true,
 				},
 			},
+			nil,
 			&exampleFieldTypes{},
 			&exampleFieldTypes{
 				F: 1.2,
@@ -201,6 +210,7 @@ func Test_Decoder(t *testing.T) {
 					true,
 				},
 			},
+			nil,
 			&examplePointerTypes{},
 			func() interface{} {
 				s := "string value"
@@ -231,6 +241,7 @@ func Test_Decoder(t *testing.T) {
 					true,
 				},
 			},
+			nil,
 			&examplePointerTypes{},
 			func() interface{} {
 				s := "string value"
@@ -255,6 +266,7 @@ func Test_Decoder(t *testing.T) {
 					1,
 				},
 			},
+			nil,
 			&exampleStructTags{},
 			&exampleStructTags{
 				S: "string value",
@@ -280,6 +292,7 @@ func Test_Decoder(t *testing.T) {
 					1,
 				},
 			},
+			nil,
 			&exampleIntConv{},
 			&exampleIntConv{
 				1, 1, 1, 1, 1,
@@ -301,6 +314,7 @@ func Test_Decoder(t *testing.T) {
 					1.0,
 				},
 			},
+			nil,
 			&exampleFieldTypes{},
 			&exampleFieldTypes{
 				F: 1.0,
@@ -322,6 +336,7 @@ func Test_Decoder(t *testing.T) {
 					1.0,
 				},
 			},
+			nil,
 			&examplePointerTypes{},
 			func() interface{} {
 				f := 1.0
@@ -340,6 +355,7 @@ func Test_Decoder(t *testing.T) {
 					([]byte)("hello world"),
 				},
 			},
+			nil,
 			&exampleBlobTypes{},
 			&exampleBlobTypes{
 				B: ([]byte)("hello world"),
@@ -356,6 +372,7 @@ func Test_Decoder(t *testing.T) {
 					([]byte)("hello world"),
 				},
 			},
+			nil,
 			&exampleJSONRawTypes{},
 			&exampleJSONRawTypes{
 				B: (json.RawMessage)("hello world"),
@@ -374,6 +391,7 @@ func Test_Decoder(t *testing.T) {
 					int(refTime.Unix()),
 				},
 			},
+			nil,
 			&exampleTimeTypes{},
 			&exampleTimeTypes{
 				T:  refTime,
@@ -393,6 +411,7 @@ func Test_Decoder(t *testing.T) {
 					int(refTime.UnixNano()),
 				},
 			},
+			nil,
 			&exampleTimeNano{},
 			&exampleTimeNano{
 				T: refTime,
@@ -411,6 +430,7 @@ func Test_Decoder(t *testing.T) {
 					"value2",
 				},
 			},
+			nil,
 			&exampleInterface{},
 			func() interface{} {
 				var x interface{}
@@ -439,6 +459,7 @@ func Test_Decoder(t *testing.T) {
 					[]byte("blob value"),
 				},
 			},
+			nil,
 			new(map[string]interface{}),
 			&map[string]interface{}{
 				"I": 1,
@@ -447,14 +468,91 @@ func Test_Decoder(t *testing.T) {
 				"B": []byte("blob value"),
 			},
 		},
+		{
+			"Decoding using type-hints",
+			testStmt{
+				columns: []string{"B", "T"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeInteger,
+					sqlite.TypeText,
+				},
+				values: []interface{}{
+					true,
+					refTime.Format(SqliteTimeFormat),
+				},
+			},
+			[]ColumnDef{
+				{
+					Name:   "B",
+					Type:   sqlite.TypeInteger,
+					GoType: reflect.TypeOf(true),
+				},
+				{
+					Name:   "T",
+					Type:   sqlite.TypeText,
+					GoType: reflect.TypeOf(time.Time{}),
+					IsTime: true,
+				},
+			},
+			new(map[string]interface{}),
+			&map[string]interface{}{
+				"B": true,
+				"T": refTime,
+			},
+		},
+		{
+			"Decoding into type aliases",
+			testStmt{
+				columns: []string{"B"},
+				types: []sqlite.ColumnType{
+					sqlite.TypeBlob,
+				},
+				values: []interface{}{
+					[]byte(`{"foo": "bar}`),
+				},
+			},
+			[]ColumnDef{
+				{
+					Name:   "B",
+					Type:   sqlite.TypeBlob,
+					GoType: reflect.TypeOf(json.RawMessage(`{"foo": "bar}`)),
+				},
+			},
+			new(map[string]interface{}),
+			&map[string]interface{}{
+				"B": json.RawMessage(`{"foo": "bar}`),
+			},
+		},
+		{
+			"Decoding into type aliases #2",
+			testStmt{
+				columns: []string{"I"},
+				types:   []sqlite.ColumnType{sqlite.TypeInteger},
+				values: []interface{}{
+					10,
+				},
+			},
+			[]ColumnDef{
+				{
+					Name:   "I",
+					Type:   sqlite.TypeInteger,
+					GoType: reflect.TypeOf(myInt(0)),
+				},
+			},
+			new(map[string]interface{}),
+			&map[string]interface{}{
+				"I": myInt(10),
+			},
+		},
 	}
 
 	for idx := range cases {
 		c := cases[idx]
 		t.Run(c.Desc, func(t *testing.T) {
-			t.Parallel()
+			//t.Parallel()
 
-			err := DecodeStmt(ctx, c.Stmt, c.Result, DefaultDecodeConfig)
+			log.Println(c.Desc)
+			err := DecodeStmt(ctx, &TableSchema{Columns: c.ColumnDef}, c.Stmt, c.Result, DefaultDecodeConfig)
 			if fn, ok := c.Expected.(func() interface{}); ok {
 				c.Expected = fn()
 			}
diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go
index 6de0c66a..88eceefd 100644
--- a/netquery/orm/query_runner.go
+++ b/netquery/orm/query_runner.go
@@ -20,6 +20,7 @@ type (
 		NamedArgs    map[string]interface{}
 		Result       interface{}
 		DecodeConfig DecodeConfig
+		Schema       TableSchema
 	}
 )
 
@@ -56,6 +57,12 @@ func WithNamedArgs(args map[string]interface{}) QueryOption {
 	}
 }
 
+func WithSchema(tbl TableSchema) QueryOption {
+	return func(opts *queryOpts) {
+		opts.Schema = tbl
+	}
+}
+
 // WithResult sets the result receiver. result is expected to
 // be a pointer to a slice of struct or map types.
 //
@@ -136,7 +143,7 @@ func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...Q
 
 			currentField = reflect.New(valElemType)
 
-			if err := DecodeStmt(ctx, stmt, currentField.Interface(), args.DecodeConfig); err != nil {
+			if err := DecodeStmt(ctx, &args.Schema, stmt, currentField.Interface(), args.DecodeConfig); err != nil {
 				return err
 			}
 
diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go
index 68783c3f..5d533be5 100644
--- a/netquery/orm/schema_builder.go
+++ b/netquery/orm/schema_builder.go
@@ -45,6 +45,7 @@ type (
 		Name          string
 		Nullable      bool
 		Type          sqlite.ColumnType
+		GoType        reflect.Type
 		Length        int
 		PrimaryKey    bool
 		AutoIncrement bool
@@ -145,6 +146,7 @@ func getColumnDef(fieldType reflect.StructField) (*ColumnDef, error) {
 		ft = fieldType.Type.Elem()
 	}
 
+	def.GoType = ft
 	kind := normalizeKind(ft.Kind())
 
 	switch kind {
diff --git a/netquery/query.go b/netquery/query.go
index 6febf0b6..929255ee 100644
--- a/netquery/query.go
+++ b/netquery/query.go
@@ -16,6 +16,12 @@ import (
 type (
 	Query map[string][]Matcher
 
+	MatchType interface {
+		Operator() string
+	}
+
+	Equal interface{}
+
 	Matcher struct {
 		Equal    interface{}   `json:"$eq,omitempty"`
 		NotEqual interface{}   `json:"$ne,omitempty"`
@@ -27,12 +33,22 @@ type (
 	Count struct {
 		As       string `json:"as"`
 		Field    string `json:"field"`
-		Distinct bool   `json:"distict"`
+		Distinct bool   `json:"distinct"`
 	}
 
+	Sum struct {
+		Condition Query  `json:"condition"`
+		As        string `json:"as"`
+		Distinct  bool   `json:"distinct"`
+	}
+
+	// NOTE: whenever adding support for new operators make sure
+	// to update UnmarshalJSON as well.
 	Select struct {
-		Field string `json:"field"`
-		Count *Count `json:"$count"`
+		Field    string  `json:"field"`
+		Count    *Count  `json:"$count,omitempty"`
+		Sum      *Sum    `json:"$sum,omitempty"`
+		Distinct *string `json:"$distinct"`
 	}
 
 	Selects []Select
@@ -45,6 +61,11 @@ type (
 
 		selectedFields    []string
 		whitelistedFields []string
+		paramMap          map[string]interface{}
+	}
+
+	QueryActiveConnectionChartPayload struct {
+		Query Query `json:"query"`
 	}
 
 	OrderBy struct {
@@ -179,15 +200,15 @@ func (match Matcher) Validate() error {
 	return nil
 }
 
-func (match Matcher) toSQLConditionClause(ctx context.Context, idx int, conjunction string, colDef orm.ColumnDef, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
+func (match Matcher) toSQLConditionClause(ctx context.Context, suffix string, conjunction string, colDef orm.ColumnDef, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
 	var (
 		queryParts []string
 		params     = make(map[string]interface{})
 		errs       = new(multierror.Error)
-		key        = fmt.Sprintf("%s%d", colDef.Name, idx)
+		key        = fmt.Sprintf("%s%s", colDef.Name, suffix)
 	)
 
-	add := func(operator, suffix string, values ...interface{}) {
+	add := func(operator, suffix string, list bool, values ...interface{}) {
 		var placeholder []string
 
 		for idx, value := range values {
@@ -204,7 +225,7 @@ func (match Matcher) toSQLConditionClause(ctx context.Context, idx int, conjunct
 			params[uniqKey] = encodedValue
 		}
 
-		if len(placeholder) == 1 {
+		if len(placeholder) == 1 && !list {
 			queryParts = append(queryParts, fmt.Sprintf("%s %s %s", colDef.Name, operator, placeholder[0]))
 		} else {
 			queryParts = append(queryParts, fmt.Sprintf("%s %s ( %s )", colDef.Name, operator, strings.Join(placeholder, ", ")))
@@ -212,23 +233,23 @@ func (match Matcher) toSQLConditionClause(ctx context.Context, idx int, conjunct
 	}
 
 	if match.Equal != nil {
-		add("=", "eq", match.Equal)
+		add("=", "eq", false, match.Equal)
 	}
 
 	if match.NotEqual != nil {
-		add("!=", "ne", match.NotEqual)
+		add("!=", "ne", false, match.NotEqual)
 	}
 
 	if match.In != nil {
-		add("IN", "in", match.In...)
+		add("IN", "in", true, match.In...)
 	}
 
 	if match.NotIn != nil {
-		add("NOT IN", "notin", match.NotIn...)
+		add("NOT IN", "notin", true, match.NotIn...)
 	}
 
 	if match.Like != "" {
-		add("LIKE", "like", match.Like)
+		add("LIKE", "like", false, match.Like)
 	}
 
 	if len(queryParts) == 0 {
@@ -244,7 +265,7 @@ func (match Matcher) toSQLConditionClause(ctx context.Context, idx int, conjunct
 	return "( " + strings.Join(queryParts, " "+conjunction+" ") + " )", params, errs.ErrorOrNil()
 }
 
-func (query Query) toSQLWhereClause(ctx context.Context, m *orm.TableSchema, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
+func (query Query) toSQLWhereClause(ctx context.Context, suffix string, m *orm.TableSchema, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
 	if len(query) == 0 {
 		return "", nil, nil
 	}
@@ -279,7 +300,7 @@ func (query Query) toSQLWhereClause(ctx context.Context, m *orm.TableSchema, enc
 
 		queryParts := make([]string, len(values))
 		for idx, val := range values {
-			matcherQuery, params, err := val.toSQLConditionClause(ctx, idx, "AND", colDef, encoderConfig)
+			matcherQuery, params, err := val.toSQLConditionClause(ctx, fmt.Sprintf("%s%d", suffix, idx), "AND", colDef, encoderConfig)
 			if err != nil {
 				errs.Errors = append(errs.Errors,
 					fmt.Errorf("invalid matcher at index %d for column %s: %w", idx, colDef.Name, err),
@@ -359,8 +380,10 @@ func (sel *Select) UnmarshalJSON(blob []byte) error {
 	// directly
 	if blob[0] == '{' {
 		var res struct {
-			Field string `json:"field"`
-			Count *Count `json:"$count"`
+			Field    string  `json:"field"`
+			Count    *Count  `json:"$count"`
+			Sum      *Sum    `json:"$sum"`
+			Distinct *string `json:"$distinct"`
 		}
 
 		if err := json.Unmarshal(blob, &res); err != nil {
@@ -369,6 +392,8 @@ func (sel *Select) UnmarshalJSON(blob []byte) error {
 
 		sel.Count = res.Count
 		sel.Field = res.Field
+		sel.Distinct = res.Distinct
+		sel.Sum = res.Sum
 
 		if sel.Count != nil && sel.Count.As != "" {
 			if !charOnlyRegexp.MatchString(sel.Count.As) {
diff --git a/netquery/query_handler.go b/netquery/query_handler.go
index a1e4353f..318e494d 100644
--- a/netquery/query_handler.go
+++ b/netquery/query_handler.go
@@ -58,6 +58,7 @@ func (qh *QueryHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 		query,
 		orm.WithNamedArgs(paramMap),
 		orm.WithResult(&result),
+		orm.WithSchema(*qh.Database.Schema),
 	); err != nil {
 		http.Error(resp, "Failed to execute query: "+err.Error(), http.StatusInternalServerError)
 
@@ -139,13 +140,14 @@ func (qh *QueryHandler) parseRequest(req *http.Request) (*QueryRequestPayload, e
 }
 
 func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.TableSchema) (string, map[string]interface{}, error) {
-	if err := req.prepareSelectedFields(schema); err != nil {
+	if err := req.prepareSelectedFields(ctx, schema); err != nil {
 		return "", nil, fmt.Errorf("perparing selected fields: %w", err)
 	}
 
 	// build the SQL where clause from the payload query
 	whereClause, paramMap, err := req.Query.toSQLWhereClause(
 		ctx,
+		"",
 		schema,
 		orm.DefaultEncodeConfig,
 	)
@@ -153,6 +155,14 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab
 		return "", nil, fmt.Errorf("ganerating where clause: %w", err)
 	}
 
+	if req.paramMap == nil {
+		req.paramMap = make(map[string]interface{})
+	}
+
+	for key, val := range paramMap {
+		req.paramMap[key] = val
+	}
+
 	// build the actual SQL query statement
 	// FIXME(ppacher): add support for group-by and sort-by
 
@@ -173,20 +183,26 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab
 	}
 	query += " " + groupByClause + " " + orderByClause
 
-	return query, paramMap, nil
+	return query, req.paramMap, nil
 }
 
-func (req *QueryRequestPayload) prepareSelectedFields(schema *orm.TableSchema) error {
-	for _, s := range req.Select {
+func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schema *orm.TableSchema) error {
+	for idx, s := range req.Select {
 		var field string
-		if s.Count != nil {
+		switch {
+		case s.Count != nil:
 			field = s.Count.Field
-		} else {
+		case s.Distinct != nil:
+			field = *s.Distinct
+		case s.Sum != nil:
+			// field is not used in case of $sum
+			field = "*"
+		default:
 			field = s.Field
 		}
 
 		colName := "*"
-		if field != "*" || s.Count == nil {
+		if field != "*" || (s.Count == nil && s.Sum == nil) {
 			var err error
 
 			colName, err = req.validateColumnName(schema, field)
@@ -195,7 +211,8 @@ func (req *QueryRequestPayload) prepareSelectedFields(schema *orm.TableSchema) e
 			}
 		}
 
-		if s.Count != nil {
+		switch {
+		case s.Count != nil:
 			var as = s.Count.As
 			if as == "" {
 				as = fmt.Sprintf("%s_count", colName)
@@ -204,9 +221,34 @@ func (req *QueryRequestPayload) prepareSelectedFields(schema *orm.TableSchema) e
 			if s.Count.Distinct {
 				distinct = "DISTINCT "
 			}
-			req.selectedFields = append(req.selectedFields, fmt.Sprintf("COUNT(%s%s) as %s", distinct, colName, as))
+			req.selectedFields = append(
+				req.selectedFields,
+				fmt.Sprintf("COUNT(%s%s) AS %s", distinct, colName, as),
+			)
 			req.whitelistedFields = append(req.whitelistedFields, as)
-		} else {
+
+		case s.Sum != nil:
+			if s.Sum.As == "" {
+				return fmt.Errorf("missing 'as' for $sum")
+			}
+
+			clause, params, err := s.Sum.Condition.toSQLWhereClause(ctx, fmt.Sprintf("sel%d", idx), schema, orm.DefaultEncodeConfig)
+			if err != nil {
+				return fmt.Errorf("in $sum: %w", err)
+			}
+
+			req.paramMap = params
+			req.selectedFields = append(
+				req.selectedFields,
+				fmt.Sprintf("SUM(%s) AS %s", clause, s.Sum.As),
+			)
+			req.whitelistedFields = append(req.whitelistedFields, s.Sum.As)
+
+		case s.Distinct != nil:
+			req.selectedFields = append(req.selectedFields, fmt.Sprintf("DISTINCT %s", colName))
+			req.whitelistedFields = append(req.whitelistedFields, colName)
+
+		default:
 			req.selectedFields = append(req.selectedFields, colName)
 		}
 	}
@@ -251,6 +293,10 @@ func (req *QueryRequestPayload) generateSelectClause() string {
 }
 
 func (req *QueryRequestPayload) generateOrderByClause(schema *orm.TableSchema) (string, error) {
+	if len(req.OrderBy) == 0 {
+		return "", nil
+	}
+
 	var orderBys = make([]string, len(req.OrderBy))
 	for idx, sort := range req.OrderBy {
 		colName, err := req.validateColumnName(schema, sort.Field)
@@ -286,7 +332,7 @@ func (req *QueryRequestPayload) validateColumnName(schema *orm.TableSchema, fiel
 		}
 	}
 
-	return "", fmt.Errorf("column name %s not allowed", field)
+	return "", fmt.Errorf("column name %q not allowed", field)
 }
 
 // compile time check
diff --git a/netquery/query_test.go b/netquery/query_test.go
index f8e8b3e4..7d9a0393 100644
--- a/netquery/query_test.go
+++ b/netquery/query_test.go
@@ -228,7 +228,7 @@ func Test_QueryBuilder(t *testing.T) {
 	for idx, c := range cases {
 		t.Run(c.N, func(t *testing.T) {
 			//t.Parallel()
-			str, params, err := c.Q.toSQLWhereClause(context.TODO(), tbl, orm.DefaultEncodeConfig)
+			str, params, err := c.Q.toSQLWhereClause(context.TODO(), "", tbl, orm.DefaultEncodeConfig)
 
 			if c.E != nil {
 				if assert.Error(t, err) {

From bef911e925003145f34dcb90a586df305567fc87 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 10 May 2022 10:04:21 +0200
Subject: [PATCH 61/86] Add support for pagination to netquery module

---
 netquery/query.go         | 31 +++++++++++++++++++++++++++++++
 netquery/query_handler.go |  5 +++--
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/netquery/query.go b/netquery/query.go
index 929255ee..2cd2d9ef 100644
--- a/netquery/query.go
+++ b/netquery/query.go
@@ -59,6 +59,8 @@ type (
 		OrderBy OrderBys `json:"orderBy"`
 		GroupBy []string `json:"groupBy"`
 
+		Pagination
+
 		selectedFields    []string
 		whitelistedFields []string
 		paramMap          map[string]interface{}
@@ -74,6 +76,11 @@ type (
 	}
 
 	OrderBys []OrderBy
+
+	Pagination struct {
+		PageSize int `json:"pageSize"`
+		Page     int `json:"page"`
+	}
 )
 
 func (query *Query) UnmarshalJSON(blob []byte) error {
@@ -156,6 +163,30 @@ func (query *Query) UnmarshalJSON(blob []byte) error {
 	return nil
 }
 
+// TODO(ppacher): right now we only support LIMIT and OFFSET for pagination but that
+// has an issue that loading the same page twice might yield different results due to
+// new records shifting the result slice. To overcome this, return a "PageToken" to the
+// user that includes the time the initial query was created so paginated queries can
+// ensure new records don't end up in the result set.
+func (page *Pagination) toSQLLimitOffsetClause() string {
+	limit := page.PageSize
+
+	// default and cap the limit to at most 100 items
+	// per page to avoid out-of-memory conditions when loading
+	// thousands of results at once.
+	if limit <= 0 || limit > 100 {
+		limit = 100
+	}
+
+	sql := fmt.Sprintf("LIMIT %d", limit)
+
+	if page.Page > 0 {
+		sql += fmt.Sprintf(" OFFSET %d", page.Page*limit)
+	}
+
+	return sql
+}
+
 func parseMatcher(raw json.RawMessage) (*Matcher, error) {
 	var m Matcher
 	if err := json.Unmarshal(raw, &m); err != nil {
diff --git a/netquery/query_handler.go b/netquery/query_handler.go
index 318e494d..672a343c 100644
--- a/netquery/query_handler.go
+++ b/netquery/query_handler.go
@@ -181,9 +181,10 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab
 	if whereClause != "" {
 		query += " WHERE " + whereClause
 	}
-	query += " " + groupByClause + " " + orderByClause
 
-	return query, req.paramMap, nil
+	query += " " + groupByClause + " " + orderByClause + " " + req.Pagination.toSQLLimitOffsetClause()
+
+	return strings.TrimSpace(query), req.paramMap, nil
 }
 
 func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schema *orm.TableSchema) error {

From 15f85b5ae94b23bf26abac50536e47be6bb907a8 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 17 May 2022 08:59:18 +0200
Subject: [PATCH 62/86] Add support for free-text search and minor bug fixes in
 netquery

---
 netquery/chart_handler.go | 26 ++++++++++++++++++++--
 netquery/database.go      |  2 +-
 netquery/manager.go       | 23 +++++++++++++++++---
 netquery/query.go         | 46 ++++++++++++++++++++++++++++++++++-----
 netquery/query_handler.go | 22 ++++++++++++++++---
 network/connection.go     |  5 ++++-
 6 files changed, 109 insertions(+), 15 deletions(-)

diff --git a/netquery/chart_handler.go b/netquery/chart_handler.go
index 22790fb5..aaff4892 100644
--- a/netquery/chart_handler.go
+++ b/netquery/chart_handler.go
@@ -99,9 +99,9 @@ WITH RECURSIVE epoch(x) AS (
 	UNION ALL
 		SELECT x+1 FROM epoch WHERE x+1 < strftime('%%s')+0
 )
-SELECT x as timestamp, COUNT(*) AS value FROM epoch
+SELECT x as timestamp, SUM(verdict IN (2, 5, 6)) AS value, SUM(verdict NOT IN (2, 5, 6)) as countBlocked FROM epoch
 	JOIN connections
-	ON strftime('%%s', connections.started)+0 <= timestamp+0 AND (connections.ended IS NULL OR strftime('%%s', connections.ended)+0 > timestamp+0)
+	ON strftime('%%s', connections.started)+0 <= timestamp+0 AND (connections.ended IS NULL OR strftime('%%s', connections.ended)+0 >= timestamp+0)
 		%s
 	GROUP BY round(timestamp/10, 0)*10;`
 
@@ -110,6 +110,28 @@ SELECT x as timestamp, COUNT(*) AS value FROM epoch
 		return "", nil, err
 	}
 
+	if params == nil {
+		params = make(map[string]interface{})
+	}
+
+	if req.TextSearch != nil {
+		textSearch, textParams, err := req.TextSearch.toSQLConditionClause(ctx, schema, "", orm.DefaultEncodeConfig)
+		if err != nil {
+			return "", nil, err
+		}
+
+		if textSearch != "" {
+			if clause != "" {
+				clause += " AND "
+			}
+			clause += textSearch
+
+			for key, val := range textParams {
+				params[key] = val
+			}
+		}
+	}
+
 	if clause == "" {
 		return fmt.Sprintf(template, ""), map[string]interface{}{}, nil
 	}
diff --git a/netquery/database.go b/netquery/database.go
index 4c53ebdd..5197868d 100644
--- a/netquery/database.go
+++ b/netquery/database.go
@@ -65,7 +65,6 @@ type (
 		// reused afterwards.
 		ID              string            `sqlite:"id,primary"`
 		ProfileID       string            `sqlite:"profile"`
-		ProfileSource   string            `sqlite:"profileSource"`
 		Path            string            `sqlite:"path"`
 		Type            string            `sqlite:"type,varchar(8)"`
 		External        bool              `sqlite:"external"`
@@ -92,6 +91,7 @@ type (
 		ExtraData       json.RawMessage   `sqlite:"extra_data"`
 		Allowed         *bool             `sqlite:"allowed"`
 		ProfileRevision int               `sqlite:"profile_revision"`
+		ExitNode        *string           `sqlite:"exit_node"`
 	}
 )
 
diff --git a/netquery/manager.go b/netquery/manager.go
index 559ccdfe..2620e510 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -174,8 +174,7 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		Internal:        conn.Internal,
 		Direction:       direction,
 		Type:            ConnectionTypeToString[conn.Type],
-		ProfileID:       conn.ProcessContext.Profile,
-		ProfileSource:   conn.ProcessContext.Source,
+		ProfileID:       conn.ProcessContext.Source + "/" + conn.ProcessContext.Profile,
 		Path:            conn.ProcessContext.BinaryPath,
 		ProfileRevision: int(conn.ProfileRevisionCounter),
 	}
@@ -191,6 +190,11 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 	case network.VerdictAccept, network.VerdictRerouteToNameserver, network.VerdictRerouteToTunnel:
 		accepted := true
 		c.Allowed = &accepted
+	case network.VerdictUndecided, network.VerdictUndeterminable:
+		c.Allowed = nil
+	default:
+		allowed := false
+		c.Allowed = &allowed
 	}
 
 	if conn.Ended > 0 {
@@ -198,12 +202,25 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		c.Ended = &ended
 	}
 
-	extraData := map[string]interface{}{}
+	extraData := map[string]interface{}{
+		"pid": conn.ProcessContext.PID,
+	}
 
 	if conn.TunnelContext != nil {
 		extraData["tunnel"] = conn.TunnelContext
+		exitNode := conn.TunnelContext.GetExitNodeID()
+		c.ExitNode = &exitNode
 	}
 
+	if conn.DNSContext != nil {
+		extraData["dns"] = conn.DNSContext
+	}
+
+	// TODO(ppacher): enable when TLS inspection is merged
+	// if conn.TLSContext != nil {
+	// 	extraData["tls"] = conn.TLSContext
+	// }
+
 	if conn.Entity != nil {
 		extraData["cname"] = conn.Entity.CNAME
 		extraData["blockedByLists"] = conn.Entity.BlockedByLists
diff --git a/netquery/query.go b/netquery/query.go
index 2cd2d9ef..58c6cdee 100644
--- a/netquery/query.go
+++ b/netquery/query.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/safing/portmaster/netquery/orm"
+	"zombiezen.com/go/sqlite"
 )
 
 type (
@@ -53,11 +54,17 @@ type (
 
 	Selects []Select
 
+	TextSearch struct {
+		Fields []string `json:"fields"`
+		Value  string   `json:"value"`
+	}
+
 	QueryRequestPayload struct {
-		Select  Selects  `json:"select"`
-		Query   Query    `json:"query"`
-		OrderBy OrderBys `json:"orderBy"`
-		GroupBy []string `json:"groupBy"`
+		Select     Selects     `json:"select"`
+		Query      Query       `json:"query"`
+		OrderBy    OrderBys    `json:"orderBy"`
+		GroupBy    []string    `json:"groupBy"`
+		TextSearch *TextSearch `json:"textSearch"`
 
 		Pagination
 
@@ -67,7 +74,8 @@ type (
 	}
 
 	QueryActiveConnectionChartPayload struct {
-		Query Query `json:"query"`
+		Query      Query       `json:"query"`
+		TextSearch *TextSearch `json:"textSearch"`
 	}
 
 	OrderBy struct {
@@ -231,6 +239,34 @@ func (match Matcher) Validate() error {
 	return nil
 }
 
+func (text TextSearch) toSQLConditionClause(ctx context.Context, schema *orm.TableSchema, suffix string, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
+	var (
+		queryParts []string
+		params     = make(map[string]interface{})
+	)
+
+	key := fmt.Sprintf(":t%s", suffix)
+	params[key] = fmt.Sprintf("%%%s%%", text.Value)
+
+	for _, field := range text.Fields {
+		colDef := schema.GetColumnDef(field)
+		if colDef == nil {
+			return "", nil, fmt.Errorf("column %s is not allowed in text-search", colDef.Name)
+		}
+		if colDef.Type != sqlite.TypeText {
+			return "", nil, fmt.Errorf("type of column %s cannot be used in text-search", colDef.Name)
+		}
+
+		queryParts = append(queryParts, fmt.Sprintf("%s LIKE %s", colDef.Name, key))
+	}
+
+	if len(queryParts) == 0 {
+		return "", nil, nil
+	}
+
+	return "( " + strings.Join(queryParts, " OR ") + " )", params, nil
+}
+
 func (match Matcher) toSQLConditionClause(ctx context.Context, suffix string, conjunction string, colDef orm.ColumnDef, encoderConfig orm.EncodeConfig) (string, map[string]interface{}, error) {
 	var (
 		queryParts []string
diff --git a/netquery/query_handler.go b/netquery/query_handler.go
index 672a343c..1b2b5411 100644
--- a/netquery/query_handler.go
+++ b/netquery/query_handler.go
@@ -152,7 +152,7 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab
 		orm.DefaultEncodeConfig,
 	)
 	if err != nil {
-		return "", nil, fmt.Errorf("ganerating where clause: %w", err)
+		return "", nil, fmt.Errorf("generating where clause: %w", err)
 	}
 
 	if req.paramMap == nil {
@@ -163,8 +163,24 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab
 		req.paramMap[key] = val
 	}
 
-	// build the actual SQL query statement
-	// FIXME(ppacher): add support for group-by and sort-by
+	if req.TextSearch != nil {
+		textClause, textParams, err := req.TextSearch.toSQLConditionClause(ctx, schema, "", orm.DefaultEncodeConfig)
+		if err != nil {
+			return "", nil, fmt.Errorf("generating text-search clause: %w", err)
+		}
+
+		if textClause != "" {
+			if whereClause != "" {
+				whereClause += " AND "
+			}
+
+			whereClause += textClause
+
+			for key, val := range textParams {
+				req.paramMap[key] = val
+			}
+		}
+	}
 
 	groupByClause, err := req.generateGroupByClause(schema)
 	if err != nil {
diff --git a/network/connection.go b/network/connection.go
index 2756cffa..57ad93d7 100644
--- a/network/connection.go
+++ b/network/connection.go
@@ -149,7 +149,10 @@ type Connection struct { //nolint:maligned // TODO: fix alignment
 	DNSContext *resolver.DNSRequestContext
 	// TunnelContext holds additional information about the tunnel that this
 	// connection is using.
-	TunnelContext interface{}
+	TunnelContext interface {
+		GetExitNodeID() string
+	}
+
 	// Internal is set to true if the connection is attributed as an
 	// Portmaster internal connection. Internal may be set at different
 	// points and access to it must be guarded by the connection lock.

From 9c0b7ddf51085fa536af97cac3cbcb40cb17359a Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 09:56:21 +0200
Subject: [PATCH 63/86] Fix direction detection for unsupported protocols

---
 network/state/lookup.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/state/lookup.go b/network/state/lookup.go
index 6e2525f1..46aac9a6 100644
--- a/network/state/lookup.go
+++ b/network/state/lookup.go
@@ -60,7 +60,7 @@ func Lookup(pktInfo *packet.Info, fast bool) (pid int, inbound bool, err error)
 		return udp6Table.lookup(pktInfo, fast)
 
 	default:
-		return socket.UndefinedProcessID, false, errors.New("unsupported protocol for finding process")
+		return socket.UndefinedProcessID, pktInfo.Inbound, errors.New("unsupported protocol for finding process")
 	}
 }
 

From bb782ba98fc23bcad1e2edfe5670d8920c9f2444 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 09:56:47 +0200
Subject: [PATCH 64/86] Add check to ensure matching PID constants

---
 process/special.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/process/special.go b/process/special.go
index 4b6a89e4..7d2c3e93 100644
--- a/process/special.go
+++ b/process/special.go
@@ -8,6 +8,7 @@ import (
 	"golang.org/x/sync/singleflight"
 
 	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/network/socket"
 	"github.com/safing/portmaster/profile"
 )
 
@@ -28,6 +29,13 @@ const (
 	NetworkHostProcessID = -255
 )
 
+func init() {
+	// Check required matching values.
+	if UndefinedProcessID != socket.UndefinedProcessID {
+		panic("UndefinedProcessID does not match socket.UndefinedProcessID")
+	}
+}
+
 var (
 	// unidentifiedProcess is used for non-attributed outgoing connections.
 	unidentifiedProcess = &Process{

From 3a98b2cc054bdb100da322e0fdd1eda108f42f7a Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 09:57:55 +0200
Subject: [PATCH 65/86] Make intel index override-able by other indexes

---
 updates/helper/indexes.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/updates/helper/indexes.go b/updates/helper/indexes.go
index db97b84d..e925be1a 100644
--- a/updates/helper/indexes.go
+++ b/updates/helper/indexes.go
@@ -33,6 +33,12 @@ func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string, delet
 	// Reset indexes before adding them (again).
 	registry.ResetIndexes()
 
+	// Add the intel index first, in order to be able to override it with the
+	// other indexes when needed.
+	registry.AddIndex(updater.Index{
+		Path: "all/intel/intel.json",
+	})
+
 	// Always add the stable index as a base.
 	registry.AddIndex(updater.Index{
 		Path: ReleaseChannelStable + ".json",
@@ -85,13 +91,6 @@ func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string, delet
 		}
 	}
 
-	// Add the intel index last, as it updates the fastest and should not be
-	// crippled by other faulty indexes. It can only specify versions for its
-	// scope anyway.
-	registry.AddIndex(updater.Index{
-		Path: "all/intel/intel.json",
-	})
-
 	// Set pre-release usage.
 	registry.SetUsePreReleases(usePreReleases)
 

From 0dce13d18fd22b29a140833816933b0ae4dff3ff Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 10:40:21 +0200
Subject: [PATCH 66/86] Fix detection of incoming localhost packets on Linux

---
 firewall/interception/nfq/packet.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/firewall/interception/nfq/packet.go b/firewall/interception/nfq/packet.go
index 46ca3ea5..528f2e3c 100644
--- a/firewall/interception/nfq/packet.go
+++ b/firewall/interception/nfq/packet.go
@@ -141,6 +141,13 @@ func (pkt *packet) Drop() error {
 }
 
 func (pkt *packet) PermanentAccept() error {
+	// If the packet is localhost only, do not permanently accept the outgoing
+	// packet, as the packet mark will be copied to the connection mark, which
+	// will stick and it will bypass the incoming queue.
+	if !pkt.Info().Inbound && pkt.Info().Dst.IsLoopback() {
+		return pkt.Accept()
+	}
+
 	return pkt.mark(MarkAcceptAlways)
 }
 

From 65974e989dd883c8b6fde8835d45b7914d279113 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 10:41:35 +0200
Subject: [PATCH 67/86] Stop using deprecated function

---
 firewall/dns.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/firewall/dns.go b/firewall/dns.go
index cb78e5c4..d8c4b582 100644
--- a/firewall/dns.go
+++ b/firewall/dns.go
@@ -263,7 +263,7 @@ func UpdateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *netw
 	// Package IPs and CNAMEs into IPInfo structs.
 	for _, ip := range ips {
 		// Never save domain attributions for localhost IPs.
-		if netutils.ClassifyIP(ip) == netutils.HostLocal {
+		if netutils.GetIPScope(ip) == netutils.HostLocal {
 			continue
 		}
 

From d40ad3125d7b00b5e3310cb71e7cba6bca910f08 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 14:33:09 +0200
Subject: [PATCH 68/86] Improve rules and filterlists config options

---
 profile/config.go | 45 ++++++++++++++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 13 deletions(-)

diff --git a/profile/config.go b/profile/config.go
index 987594d4..f03fca74 100644
--- a/profile/config.go
+++ b/profile/config.go
@@ -258,6 +258,12 @@ Examples: "192.168.0.1 TCP/HTTP", "LAN UDP/50000-55000", "example.com */HTTPS",
 Important: DNS Requests are only matched against domain and filter list rules, all others require an IP address and are checked only with the following IP connection.
 `, `"`, "`")
 
+	// rulesVerdictNames defines the verdicts names to be used for filter rules.
+	rulesVerdictNames := map[string]string{
+		"-": "Block", // Default.
+		"+": "Allow",
+	}
+
 	// Endpoint Filter List
 	err = config.Register(&config.Option{
 		Name:         "Outgoing Rules",
@@ -268,10 +274,11 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 		OptType:      config.OptTypeStringArray,
 		DefaultValue: []string{},
 		Annotations: config.Annotations{
-			config.StackableAnnotation:    true,
-			config.DisplayHintAnnotation:  endpoints.DisplayHintEndpointList,
-			config.DisplayOrderAnnotation: cfgOptionEndpointsOrder,
-			config.CategoryAnnotation:     "Rules",
+			config.StackableAnnotation:                   true,
+			config.DisplayHintAnnotation:                 endpoints.DisplayHintEndpointList,
+			config.DisplayOrderAnnotation:                cfgOptionEndpointsOrder,
+			config.CategoryAnnotation:                    "Rules",
+			endpoints.EndpointListVerdictNamesAnnotation: rulesVerdictNames,
 		},
 		ValidationRegex: endpoints.ListEntryValidationRegex,
 		ValidationFunc:  endpoints.ValidateEndpointListConfigOption,
@@ -283,6 +290,7 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 	cfgStringArrayOptions[CfgOptionEndpointsKey] = cfgOptionEndpoints
 
 	// Service Endpoint Filter List
+	defaultIncomingRulesValue := []string{"+ Localhost"}
 	err = config.Register(&config.Option{
 		Name:           "Incoming Rules",
 		Key:            CfgOptionServiceEndpointsKey,
@@ -290,13 +298,14 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 		Help:           rulesHelp,
 		Sensitive:      true,
 		OptType:        config.OptTypeStringArray,
-		DefaultValue:   []string{"+ Localhost"},
+		DefaultValue:   defaultIncomingRulesValue,
 		ExpertiseLevel: config.ExpertiseLevelExpert,
 		Annotations: config.Annotations{
-			config.StackableAnnotation:    true,
-			config.DisplayHintAnnotation:  endpoints.DisplayHintEndpointList,
-			config.DisplayOrderAnnotation: cfgOptionServiceEndpointsOrder,
-			config.CategoryAnnotation:     "Rules",
+			config.StackableAnnotation:                   true,
+			config.DisplayHintAnnotation:                 endpoints.DisplayHintEndpointList,
+			config.DisplayOrderAnnotation:                cfgOptionServiceEndpointsOrder,
+			config.CategoryAnnotation:                    "Rules",
+			endpoints.EndpointListVerdictNamesAnnotation: rulesVerdictNames,
 			config.QuickSettingsAnnotation: []config.QuickSetting{
 				{
 					Name:   "SSH",
@@ -313,6 +322,16 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 					Action: config.QuickMergeTop,
 					Value:  []string{"+ * */3389"},
 				},
+				{
+					Name:   "Allow all from LAN",
+					Action: config.QuickMergeTop,
+					Value:  []string{"+ LAN"},
+				},
+				{
+					Name:   "Allow all from Internet",
+					Action: config.QuickMergeTop,
+					Value:  []string{"+ Internet"},
+				},
 			},
 		},
 		ValidationRegex: endpoints.ListEntryValidationRegex,
@@ -321,7 +340,7 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 	if err != nil {
 		return err
 	}
-	cfgOptionServiceEndpoints = config.Concurrent.GetAsStringArray(CfgOptionServiceEndpointsKey, []string{})
+	cfgOptionServiceEndpoints = config.Concurrent.GetAsStringArray(CfgOptionServiceEndpointsKey, defaultIncomingRulesValue)
 	cfgStringArrayOptions[CfgOptionServiceEndpointsKey] = cfgOptionServiceEndpoints
 
 	filterListsHelp := strings.ReplaceAll(`Filter lists contain domains and IP addresses that are known to be used adversarial. The data is collected from many public sources and put into the following categories. In order to active a category, add it's "ID" to the list.
@@ -346,13 +365,13 @@ The lists are automatically updated every hour using incremental updates.
 `, `"`, "`")
 
 	// Filter list IDs
+	defaultFilterListsValue := []string{"TRAC", "MAL", "BAD"}
 	err = config.Register(&config.Option{
 		Name:         "Filter Lists",
 		Key:          CfgOptionFilterListsKey,
 		Description:  "Block connections that match enabled filter lists.",
-		Help:         filterListsHelp,
 		OptType:      config.OptTypeStringArray,
-		DefaultValue: []string{"TRAC", "MAL", "BAD"},
+		DefaultValue: defaultFilterListsValue,
 		Annotations: config.Annotations{
 			config.DisplayHintAnnotation:  "filter list",
 			config.DisplayOrderAnnotation: cfgOptionFilterListsOrder,
@@ -363,7 +382,7 @@ The lists are automatically updated every hour using incremental updates.
 	if err != nil {
 		return err
 	}
-	cfgOptionFilterLists = config.Concurrent.GetAsStringArray(CfgOptionFilterListsKey, []string{})
+	cfgOptionFilterLists = config.Concurrent.GetAsStringArray(CfgOptionFilterListsKey, defaultFilterListsValue)
 	cfgStringArrayOptions[CfgOptionFilterListsKey] = cfgOptionFilterLists
 
 	// Include CNAMEs

From 6339eb0c49e01ea627319f25e6725f9495291876 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 14:33:28 +0200
Subject: [PATCH 69/86] Remove filter lists help

---
 profile/config.go | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/profile/config.go b/profile/config.go
index f03fca74..e371a175 100644
--- a/profile/config.go
+++ b/profile/config.go
@@ -343,27 +343,6 @@ Important: DNS Requests are only matched against domain and filter list rules, a
 	cfgOptionServiceEndpoints = config.Concurrent.GetAsStringArray(CfgOptionServiceEndpointsKey, defaultIncomingRulesValue)
 	cfgStringArrayOptions[CfgOptionServiceEndpointsKey] = cfgOptionServiceEndpoints
 
-	filterListsHelp := strings.ReplaceAll(`Filter lists contain domains and IP addresses that are known to be used adversarial. The data is collected from many public sources and put into the following categories. In order to active a category, add it's "ID" to the list.
-
-**Ads & Trackers** - ID: "TRAC"  
-Services that track and profile people online, including as ads, analytics and telemetry.
-
-**Malware** - ID: "MAL"  
-Services that are (ab)used for attacking devices through technical means.
-
-**Deception** - ID: "DECEP"  
-Services that trick humans into thinking the service is genuine, while it is not, including phishing, fake news and fraud.
-
-**Bad Stuff (Mixed)** - ID: "BAD"  
-Miscellaneous services that are believed to be harmful to security or privacy, but their exact use is unknown, not categorized, or lists have mixed categories.
-
-**NSFW** - ID: "NSFW"  
-Services that are generally not accepted in work environments, including pornography, violence and gambling.
-
-The lists are automatically updated every hour using incremental updates.  
-[See here](https://github.com/safing/intel-data) for more detail about these lists, their sources and how to help to improve them.
-`, `"`, "`")
-
 	// Filter list IDs
 	defaultFilterListsValue := []string{"TRAC", "MAL", "BAD"}
 	err = config.Register(&config.Option{

From dac6b3c6fe31bb9e333c18abcaa3e5eac28163a9 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Wed, 22 Jun 2022 14:49:36 +0200
Subject: [PATCH 70/86] Fix linter error

---
 netenv/main.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/netenv/main.go b/netenv/main.go
index 232696cd..3363754a 100644
--- a/netenv/main.go
+++ b/netenv/main.go
@@ -1,9 +1,10 @@
 package netenv
 
 import (
+	"github.com/tevino/abool"
+
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
-	"github.com/tevino/abool"
 )
 
 // Event Names.

From 9f4e921609e65f4f2a08127249f013691075f7ba Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 21 Jun 2022 16:55:19 +0200
Subject: [PATCH 71/86] Simplify and improve update version export

---
 updates/export.go | 179 +++++++++++++++++++++++-----------------------
 updates/main.go   |  12 +++-
 2 files changed, 98 insertions(+), 93 deletions(-)

diff --git a/updates/export.go b/updates/export.go
index e00dee33..0c03b365 100644
--- a/updates/export.go
+++ b/updates/export.go
@@ -2,11 +2,8 @@ package updates
 
 import (
 	"context"
-	"errors"
 	"sync"
 
-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/database/query"
 	"github.com/safing/portbase/database/record"
 	"github.com/safing/portbase/info"
 	"github.com/safing/portbase/log"
@@ -14,48 +11,95 @@ import (
 	"github.com/safing/portmaster/updates/helper"
 )
 
-// Database key for update information.
 const (
+	// versionsDBKey is the database key for update version information.
 	versionsDBKey = "core:status/versions"
+
+	// versionsDBKey is the database key for simple update version information.
+	simpleVersionsDBKey = "core:status/simple-versions"
 )
 
-var (
-	versionExport   *versions
-	versionExportDB = database.NewInterface(&database.Options{
-		Local:    true,
-		Internal: true,
-	})
-	versionExportHook *database.RegisteredHook
-)
-
-// versions holds updates status information.
-type versions struct {
+// Versions holds update versions and status information.
+type Versions struct {
 	record.Base
-	lock sync.Mutex
+	sync.Mutex
 
 	Core      *info.Info
 	Resources map[string]*updater.Resource
 	Channel   string
 	Beta      bool
 	Staging   bool
+}
 
-	internalSave bool
+// SimpleVersions holds simplified update versions and status information.
+type SimpleVersions struct {
+	record.Base
+	sync.Mutex
+
+	Build     *info.Info
+	Resources map[string]*SimplifiedResourceVersion
+	Channel   string
+}
+
+// SimplifiedResourceVersion holds version information about one resource.
+type SimplifiedResourceVersion struct {
+	Version string
+}
+
+// GetVersions returns the update versions and status information.
+// Resources must be locked when accessed.
+func GetVersions() *Versions {
+	return &Versions{
+		Core:      info.GetInfo(),
+		Resources: registry.Export(),
+		Channel:   initialReleaseChannel,
+		Beta:      initialReleaseChannel == helper.ReleaseChannelBeta,
+		Staging:   initialReleaseChannel == helper.ReleaseChannelStaging,
+	}
+}
+
+// GetSimpleVersions returns the simplified update versions and status information.
+func GetSimpleVersions() *SimpleVersions {
+	// Fill base info.
+	v := &SimpleVersions{
+		Build:     info.GetInfo(),
+		Resources: make(map[string]*SimplifiedResourceVersion),
+		Channel:   initialReleaseChannel,
+	}
+
+	// Iterate through all versions and add version info.
+	for id, resource := range registry.Export() {
+		func() {
+			resource.Lock()
+			defer resource.Unlock()
+
+			// Get current in-used or selected version.
+			var rv *updater.ResourceVersion
+			switch {
+			case resource.ActiveVersion != nil:
+				rv = resource.ActiveVersion
+			case resource.SelectedVersion != nil:
+				rv = resource.SelectedVersion
+			}
+
+			// Get information from resource.
+			if rv != nil {
+				v.Resources[id] = &SimplifiedResourceVersion{
+					Version: rv.VersionNumber,
+				}
+			}
+		}()
+	}
+
+	return v
 }
 
 func initVersionExport() (err error) {
-	// init export struct
-	versionExport = &versions{
-		internalSave: true,
-		Channel:      initialReleaseChannel,
-		Beta:         initialReleaseChannel == helper.ReleaseChannelBeta,
-		Staging:      initialReleaseChannel == helper.ReleaseChannelStaging,
+	if err := GetVersions().save(); err != nil {
+		log.Warningf("updates: failed to export version information: %s", err)
 	}
-	versionExport.SetKey(versionsDBKey)
-
-	// attach hook to database
-	versionExportHook, err = database.RegisterHook(query.New(versionsDBKey), &exportHook{})
-	if err != nil {
-		return err
+	if err := GetSimpleVersions().save(); err != nil {
+		log.Warningf("updates: failed to export version information: %s", err)
 	}
 
 	return module.RegisterEventHook(
@@ -66,71 +110,24 @@ func initVersionExport() (err error) {
 	)
 }
 
-func stopVersionExport() error {
-	return versionExportHook.Cancel()
+func (v *Versions) save() error {
+	if !v.KeyIsSet() {
+		v.SetKey(versionsDBKey)
+	}
+	return db.Put(v)
+}
+
+func (v *SimpleVersions) save() error {
+	if !v.KeyIsSet() {
+		v.SetKey(simpleVersionsDBKey)
+	}
+	return db.Put(v)
 }
 
 // export is an event hook.
 func export(_ context.Context, _ interface{}) error {
-	// populate
-	versionExport.lock.Lock()
-	versionExport.Core = info.GetInfo()
-	versionExport.Resources = registry.Export()
-	versionExport.lock.Unlock()
-
-	// save
-	err := versionExportDB.Put(versionExport)
-	if err != nil {
-		log.Warningf("updates: failed to export versions: %s", err)
+	if err := GetVersions().save(); err != nil {
+		return err
 	}
-
-	return nil
-}
-
-// Lock locks the versionExport and all associated resources.
-func (v *versions) Lock() {
-	// lock self
-	v.lock.Lock()
-
-	// lock all resources
-	for _, res := range v.Resources {
-		res.Lock()
-	}
-}
-
-// Lock unlocks the versionExport and all associated resources.
-func (v *versions) Unlock() {
-	// unlock all resources
-	for _, res := range v.Resources {
-		res.Unlock()
-	}
-
-	// unlock self
-	v.lock.Unlock()
-}
-
-type exportHook struct {
-	database.HookBase
-}
-
-// UsesPrePut implements the Hook interface.
-func (eh *exportHook) UsesPrePut() bool {
-	return true
-}
-
-var errInternalRecord = errors.New("may not modify internal record")
-
-// PrePut implements the Hook interface.
-func (eh *exportHook) PrePut(r record.Record) (record.Record, error) {
-	if r.IsWrapped() {
-		return nil, errInternalRecord
-	}
-	ve, ok := r.(*versions)
-	if !ok {
-		return nil, errInternalRecord
-	}
-	if !ve.internalSave {
-		return nil, errInternalRecord
-	}
-	return r, nil
+	return GetSimpleVersions().save()
 }
diff --git a/updates/main.go b/updates/main.go
index 550e23ac..bee808a0 100644
--- a/updates/main.go
+++ b/updates/main.go
@@ -7,6 +7,7 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/dataroot"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
@@ -48,6 +49,11 @@ var (
 	updateASAP          bool
 	disableTaskSchedule bool
 
+	db = database.NewInterface(&database.Options{
+		Local:    true,
+		Internal: true,
+	})
+
 	// UserAgent is an HTTP User-Agent that is used to add
 	// more context to requests made by the registry when
 	// fetching resources from the update server.
@@ -55,6 +61,8 @@ var (
 )
 
 const (
+	updatesDirName = "updates"
+
 	updateFailed  = "updates:failed"
 	updateSuccess = "updates:success"
 )
@@ -108,7 +116,7 @@ func start() error {
 		registry.UserAgent = userAgentFromFlag
 	}
 	// initialize
-	err := registry.Initialize(dataroot.Root().ChildDir("updates", 0o0755))
+	err := registry.Initialize(dataroot.Root().ChildDir(updatesDirName, 0o0755))
 	if err != nil {
 		return err
 	}
@@ -275,7 +283,7 @@ func stop() error {
 		}
 	}
 
-	return stopVersionExport()
+	return nil
 }
 
 // RootPath returns the root path used for storing updates.

From 959bb012b87b7b8e05053ac1eb6141dd234e6015 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 21 Jun 2022 16:56:00 +0200
Subject: [PATCH 72/86] Add broadcasts module

---
 broadcasts/api.go          | 116 +++++++++++++++
 broadcasts/data.go         | 102 +++++++++++++
 broadcasts/install_info.go | 175 +++++++++++++++++++++++
 broadcasts/module.go       |  46 ++++++
 broadcasts/notify.go       | 285 +++++++++++++++++++++++++++++++++++++
 broadcasts/state.go        |  64 +++++++++
 core/core.go               |   3 +-
 7 files changed, 790 insertions(+), 1 deletion(-)
 create mode 100644 broadcasts/api.go
 create mode 100644 broadcasts/data.go
 create mode 100644 broadcasts/install_info.go
 create mode 100644 broadcasts/module.go
 create mode 100644 broadcasts/notify.go
 create mode 100644 broadcasts/state.go

diff --git a/broadcasts/api.go b/broadcasts/api.go
new file mode 100644
index 00000000..f855ddfc
--- /dev/null
+++ b/broadcasts/api.go
@@ -0,0 +1,116 @@
+package broadcasts
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/accessor"
+)
+
+func registerAPIEndpoints() error {
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:        `broadcasts/matching-data`,
+		Read:        api.PermitAdmin,
+		BelongsTo:   module,
+		StructFunc:  handleMatchingData,
+		Name:        "Get Broadcast Notifications Matching Data",
+		Description: "Returns the data used by the broadcast notifications to match the instance.",
+	}); err != nil {
+		return err
+	}
+
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:        `broadcasts/reset-state`,
+		Write:       api.PermitAdmin,
+		WriteMethod: http.MethodPost,
+		BelongsTo:   module,
+		ActionFunc:  handleResetState,
+		Name:        "Resets the Broadcast Notification States",
+		Description: "Delete the cache of Broadcast Notifications, making them appear again.",
+	}); err != nil {
+		return err
+	}
+
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Path:        `broadcasts/simulate`,
+		Write:       api.PermitAdmin,
+		WriteMethod: http.MethodPost,
+		BelongsTo:   module,
+		ActionFunc:  handleSimulate,
+		Name:        "Simulate Broadcast Notifications",
+		Description: "Test broadcast notifications by sending a valid source file in the body.",
+		Parameters: []api.Parameter{
+			{
+				Method:      http.MethodPost,
+				Field:       "state",
+				Value:       "true",
+				Description: "Check against state when deciding to display a broadcast notification. Acknowledgements are always saved.",
+			},
+		},
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func handleMatchingData(ar *api.Request) (i interface{}, err error) {
+	return collectData(), nil
+}
+
+func handleResetState(ar *api.Request) (msg string, err error) {
+	err = db.Delete(broadcastStatesDBKey)
+	if err != nil {
+		return "", err
+	}
+	return "Reset complete.", nil
+}
+
+func handleSimulate(ar *api.Request) (msg string, err error) {
+	// Parse broadcast notification data.
+	broadcasts, err := parseBroadcastSource(ar.InputData)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse broadcast notifications update: %w", err)
+	}
+
+	// Get and marshal matching data.
+	matchingData := collectData()
+	matchingJSON, err := json.Marshal(matchingData)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal broadcast notifications matching data: %w", err)
+	}
+	matchingDataAccessor := accessor.NewJSONBytesAccessor(&matchingJSON)
+
+	var bss *BroadcastStates
+	if ar.URL.Query().Get("state") == "true" {
+		// Get broadcast notification states.
+		bss, err = getBroadcastStates()
+		if err != nil {
+			if !errors.Is(err, database.ErrNotFound) {
+				return "", fmt.Errorf("failed to get broadcast notifications states: %w", err)
+			}
+			bss = newBroadcastStates()
+		}
+	}
+
+	// Go through all broadcast nofications and check if they match.
+	var results []string
+	for _, bn := range broadcasts.Notifications {
+		err := handleBroadcast(bn, matchingDataAccessor, bss)
+		switch {
+		case err == nil:
+			results = append(results, fmt.Sprintf("%30s: displayed", bn.id))
+		case errors.Is(err, ErrSkip):
+			results = append(results, fmt.Sprintf("%30s: %s", bn.id, err))
+		default:
+			results = append(results, fmt.Sprintf("FAILED %23s: %s", bn.id, err))
+		}
+	}
+
+	return strings.Join(results, "\n"), nil
+}
diff --git a/broadcasts/data.go b/broadcasts/data.go
new file mode 100644
index 00000000..73da1bb9
--- /dev/null
+++ b/broadcasts/data.go
@@ -0,0 +1,102 @@
+package broadcasts
+
+import (
+	"time"
+
+	"github.com/safing/portbase/config"
+	"github.com/safing/portmaster/intel/geoip"
+	"github.com/safing/portmaster/netenv"
+	"github.com/safing/portmaster/updates"
+	"github.com/safing/spn/access"
+	"github.com/safing/spn/captain"
+)
+
+var portmasterStarted = time.Now()
+
+func collectData() interface{} {
+	data := make(map[string]interface{})
+
+	// Get data about versions.
+	versions := updates.GetSimpleVersions()
+	data["Updates"] = versions
+	data["Version"] = versions.Build.Version
+	numericVersion, err := MakeNumericVersion(versions.Build.Version)
+	if err != nil {
+		data["NumericVersion"] = &DataError{
+			Error: err,
+		}
+	} else {
+		data["NumericVersion"] = numericVersion
+	}
+
+	// Get data about install.
+	installInfo, err := GetInstallInfo()
+	if err != nil {
+		data["Install"] = &DataError{
+			Error: err,
+		}
+	} else {
+		data["Install"] = installInfo
+	}
+
+	// Get global configuration.
+	data["Config"] = config.GetActiveConfigValues()
+
+	// Get data about device location.
+	locs, ok := netenv.GetInternetLocation()
+	if ok && locs.Best().LocationOrNil() != nil {
+		loc := locs.Best()
+		data["Location"] = &Location{
+			Country:        loc.Location.Country.ISOCode,
+			Coordinates:    loc.Location.Coordinates,
+			ASN:            loc.Location.AutonomousSystemNumber,
+			ASOrg:          loc.Location.AutonomousSystemOrganization,
+			Source:         loc.Source,
+			SourceAccuracy: loc.SourceAccuracy,
+		}
+	}
+
+	// Get data about SPN status.
+	data["SPN"] = captain.GetSPNStatus()
+
+	// Get data about account.
+	userRecord, err := access.GetUser()
+	if err != nil {
+		data["Account"] = &DataError{
+			Error: err,
+		}
+	} else {
+		data["Account"] = &Account{
+			UserRecord: userRecord,
+			UpToDate:   userRecord.Meta().Modified > time.Now().Add(-7*24*time.Hour).Unix(),
+			MayUseUSP:  userRecord.MayUseSPN(),
+		}
+	}
+
+	// Time running.
+	data["UptimeHours"] = int(time.Since(portmasterStarted).Hours())
+
+	return data
+}
+
+// Location holds location matching data.
+type Location struct {
+	Country        string
+	Coordinates    geoip.Coordinates
+	ASN            uint
+	ASOrg          string
+	Source         netenv.DeviceLocationSource
+	SourceAccuracy int
+}
+
+// Account holds SPN account matching data.
+type Account struct {
+	*access.UserRecord
+	UpToDate  bool
+	MayUseUSP bool
+}
+
+// DataError represents an error getting some matching data.
+type DataError struct {
+	Error error
+}
diff --git a/broadcasts/install_info.go b/broadcasts/install_info.go
new file mode 100644
index 00000000..2f667a17
--- /dev/null
+++ b/broadcasts/install_info.go
@@ -0,0 +1,175 @@
+package broadcasts
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+
+	semver "github.com/hashicorp/go-version"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/query"
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/log"
+)
+
+const installInfoDBKey = "core:status/install-info"
+
+// InstallInfo holds generic info about the install.
+type InstallInfo struct {
+	record.Base
+	sync.Mutex
+
+	Version        string
+	NumericVersion int64
+
+	Time             time.Time
+	NumericDate      int64
+	DaysSinceInstall int64
+	UnixTimestamp    int64
+}
+
+// GetInstallInfo returns the install info from the database.
+func GetInstallInfo() (*InstallInfo, error) {
+	r, err := db.Get(installInfoDBKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unwrap.
+	if r.IsWrapped() {
+		// Only allocate a new struct, if we need it.
+		newRecord := &InstallInfo{}
+		err = record.Unwrap(r, newRecord)
+		if err != nil {
+			return nil, err
+		}
+		return newRecord, nil
+	}
+
+	// or adjust type
+	newRecord, ok := r.(*InstallInfo)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *InstallInfo, but %T", r)
+	}
+	return newRecord, nil
+}
+
+func ensureInstallInfo() {
+	// Get current install info from database.
+	installInfo, err := GetInstallInfo()
+	if err != nil {
+		installInfo = &InstallInfo{}
+		if !errors.Is(err, database.ErrNotFound) {
+			log.Warningf("updates: failed to load install info: %s", err)
+		}
+	}
+
+	// Fill in missing data and save.
+	installInfo.checkAll()
+	if err := installInfo.save(); err != nil {
+		log.Warningf("updates: failed to save install info: %s", err)
+	}
+}
+
+func (ii *InstallInfo) save() error {
+	if !ii.KeyIsSet() {
+		ii.SetKey(installInfoDBKey)
+	}
+	return db.Put(ii)
+}
+
+func (ii *InstallInfo) checkAll() {
+	ii.checkVersion()
+	ii.checkInstallDate()
+}
+
+func (ii *InstallInfo) checkVersion() {
+	// Check if everything is present.
+	if ii.Version != "" && ii.NumericVersion > 0 {
+		return
+	}
+
+	// Update version information.
+	versionInfo := info.GetInfo()
+	ii.Version = versionInfo.Version
+
+	// Update numeric version.
+	if versionInfo.Version != "" {
+		numericVersion, err := MakeNumericVersion(versionInfo.Version)
+		if err != nil {
+			log.Warningf("updates: failed to make numeric version: %s", err)
+		} else {
+			ii.NumericVersion = numericVersion
+		}
+	}
+}
+
+// MakeNumericVersion makes a numeric version with the first three version
+// segment always using three digits.
+func MakeNumericVersion(version string) (numericVersion int64, err error) {
+	// Parse version string.
+	ver, err := semver.NewVersion(version)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse core version: %w", err)
+	}
+
+	// Transform version for numeric representation.
+	segments := ver.Segments()
+	for i := 0; i < 3 && i < len(segments); i++ {
+		segmentNumber := int64(segments[i])
+		if segmentNumber > 999 {
+			segmentNumber = 999
+		}
+		switch i {
+		case 0:
+			numericVersion += segmentNumber * 1000000
+		case 1:
+			numericVersion += segmentNumber * 1000
+		case 2:
+			numericVersion += segmentNumber
+		}
+	}
+
+	return numericVersion, nil
+}
+
+func (ii *InstallInfo) checkInstallDate() {
+	// Check if everything is present.
+	if ii.UnixTimestamp > 0 &&
+		ii.NumericDate > 0 &&
+		ii.DaysSinceInstall > 0 &&
+		!ii.Time.IsZero() {
+		return
+	}
+
+	// Find oldest created database entry and use it as install time.
+	oldest := time.Now().Unix()
+	it, err := db.Query(query.New("core"))
+	if err != nil {
+		log.Warningf("updates: failed to create iterator for searching DB for install time: %s", err)
+		return
+	}
+	defer it.Cancel()
+	for r := range it.Next {
+		if oldest > r.Meta().Created {
+			oldest = r.Meta().Created
+		}
+	}
+
+	// Set data.
+	ii.UnixTimestamp = oldest
+	ii.Time = time.Unix(oldest, 0)
+	ii.DaysSinceInstall = int64(time.Since(ii.Time).Hours()) / 24
+
+	// Transform date for numeric representation.
+	numericDate, err := strconv.ParseInt(ii.Time.Format("20060102"), 10, 64)
+	if err != nil {
+		log.Warningf("updates: failed to make numeric date from %s: %s", ii.Time, err)
+	} else {
+		ii.NumericDate = numericDate
+	}
+}
diff --git a/broadcasts/module.go b/broadcasts/module.go
new file mode 100644
index 00000000..360bc912
--- /dev/null
+++ b/broadcasts/module.go
@@ -0,0 +1,46 @@
+package broadcasts
+
+import (
+	"sync"
+	"time"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/modules"
+)
+
+var (
+	module *modules.Module
+
+	db = database.NewInterface(&database.Options{
+		Local:    true,
+		Internal: true,
+	})
+
+	startOnce sync.Once
+)
+
+func init() {
+	module = modules.Register("broadcasts", prep, start, nil, "updates", "netenv", "notifications")
+}
+
+func prep() error {
+	// Register API endpoints.
+	if err := registerAPIEndpoints(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func start() error {
+	// Ensure the install info is up to date.
+	ensureInstallInfo()
+
+	// Start broadcast notifier task.
+	startOnce.Do(func() {
+		module.NewTask("broadcast notifier", broadcastNotify).
+			Repeat(10 * time.Minute).Queue()
+	})
+
+	return nil
+}
diff --git a/broadcasts/notify.go b/broadcasts/notify.go
new file mode 100644
index 00000000..c7b7d661
--- /dev/null
+++ b/broadcasts/notify.go
@@ -0,0 +1,285 @@
+package broadcasts
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ghodss/yaml"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/accessor"
+	"github.com/safing/portbase/database/query"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portbase/notifications"
+	"github.com/safing/portmaster/updates"
+)
+
+const (
+	broadcastsResourcePath = "intel/portmaster/notifications.yaml"
+
+	broadcastNotificationIDPrefix = "broadcasts:"
+
+	minRepeatDuration = 1 * time.Hour
+)
+
+// Errors.
+var (
+	ErrSkip                  = errors.New("broadcast skipped")
+	ErrSkipDoesNotMatch      = fmt.Errorf("%w: does not match", ErrSkip)
+	ErrSkipAlreadyActive     = fmt.Errorf("%w: already active", ErrSkip)
+	ErrSkipAlreadyShown      = fmt.Errorf("%w: already shown", ErrSkip)
+	ErrSkipRemovedByMismatch = fmt.Errorf("%w: removed due to mismatch", ErrSkip)
+	ErrSkipRemovedBySource   = fmt.Errorf("%w: removed by source", ErrSkip)
+)
+
+// BroadcastNotifications holds the data structure of the broadcast
+// notifications update file.
+type BroadcastNotifications struct {
+	Notifications map[string]*BroadcastNotification
+}
+
+// BroadcastNotification is a single broadcast notification.
+type BroadcastNotification struct {
+	*notifications.Notification
+	id string
+
+	// Match holds a query string that needs to match the local matching data in
+	// order for the broadcast to be displayed.
+	Match         string
+	matchingQuery *query.Query
+	// AttachToModule signifies if the broadcast notification should be attached to the module.
+	AttachToModule bool
+	// Remove signifies that the broadcast should be canceled and its state removed.
+	Remove bool
+	// Permanent signifies that the broadcast cannot be acknowledge by the user
+	// and remains in the UI indefinitely.
+	Permanent bool
+	// Repeat specifies a duration after which the broadcast should be shown again.
+	Repeat         string
+	repeatDuration time.Duration
+}
+
+func broadcastNotify(ctx context.Context, t *modules.Task) error {
+	// Get broadcast notifications file, load it from disk and parse it.
+	broadcastsResource, err := updates.GetFile(broadcastsResourcePath)
+	if err != nil {
+		return fmt.Errorf("failed to get broadcast notifications update: %w", err)
+	}
+	broadcastsData, err := ioutil.ReadFile(broadcastsResource.Path())
+	if err != nil {
+		return fmt.Errorf("failed to load broadcast notifications update: %w", err)
+	}
+	broadcasts, err := parseBroadcastSource(broadcastsData)
+	if err != nil {
+		return fmt.Errorf("failed to parse broadcast notifications update: %w", err)
+	}
+
+	// Get and marshal matching data.
+	matchingData := collectData()
+	matchingJSON, err := json.Marshal(matchingData)
+	if err != nil {
+		return fmt.Errorf("failed to marshal broadcast notifications matching data: %w", err)
+	}
+	matchingDataAccessor := accessor.NewJSONBytesAccessor(&matchingJSON)
+
+	// Get broadcast notification states.
+	bss, err := getBroadcastStates()
+	if err != nil {
+		if !errors.Is(err, database.ErrNotFound) {
+			return fmt.Errorf("failed to get broadcast notifications states: %w", err)
+		}
+		bss = newBroadcastStates()
+	}
+
+	// Go through all broadcast nofications and check if they match.
+	for _, bn := range broadcasts.Notifications {
+		err := handleBroadcast(bn, matchingDataAccessor, bss)
+		switch {
+		case err == nil:
+			log.Infof("broadcasts: displaying broadcast %s", bn.id)
+		case errors.Is(err, ErrSkip):
+			log.Tracef("broadcasts: skipped displaying broadcast %s: %s", bn.id, err)
+		default:
+			log.Warningf("broadcasts: failed to handle broadcast %s: %s", bn.id, err)
+		}
+	}
+
+	return nil
+}
+
+func parseBroadcastSource(yamlData []byte) (*BroadcastNotifications, error) {
+	// Parse data.
+	broadcasts := &BroadcastNotifications{}
+	err := yaml.Unmarshal(yamlData, broadcasts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add IDs to struct for easier handling.
+	for id, bn := range broadcasts.Notifications {
+		bn.id = id
+
+		// Parse matching query.
+		if bn.Match != "" {
+			q, err := query.ParseQuery("query / where " + bn.Match)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse query of broadcast notification %s: %w", bn.id, err)
+			}
+			bn.matchingQuery = q
+		}
+
+		// Parse the repeat duration.
+		if bn.Repeat != "" {
+			duration, err := time.ParseDuration(bn.Repeat)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse repeat duration of broadcast notification %s: %w", bn.id, err)
+			}
+			bn.repeatDuration = duration
+			// Raise duration to minimum.
+			if bn.repeatDuration < minRepeatDuration {
+				bn.repeatDuration = minRepeatDuration
+			}
+		}
+	}
+
+	return broadcasts, nil
+}
+
+func handleBroadcast(bn *BroadcastNotification, matchingDataAccessor accessor.Accessor, bss *BroadcastStates) error {
+	// Check if broadcast was already shown.
+	if bss != nil {
+		state, ok := bss.States[bn.id]
+		switch {
+		case !ok || state.Read.IsZero():
+			// Was never shown, continue.
+		case bn.repeatDuration == 0 && !state.Read.IsZero():
+			// Was already shown and is not repeated, skip.
+			return ErrSkipAlreadyShown
+		case bn.repeatDuration > 0 && time.Now().Add(-bn.repeatDuration).After(state.Read):
+			// Was already shown, but should be repeated now, continue.
+		}
+	}
+
+	// Check if broadcast should be removed.
+	if bn.Remove {
+		removeBroadcast(bn, bss)
+		return ErrSkipRemovedBySource
+	}
+
+	// Skip if broadcast does not match.
+	if bn.matchingQuery != nil && !bn.matchingQuery.MatchesAccessor(matchingDataAccessor) {
+		removed := removeBroadcast(bn, bss)
+		if removed {
+			return ErrSkipRemovedByMismatch
+		}
+		return ErrSkipDoesNotMatch
+	}
+
+	// Check if there is already an active notification for this.
+	eventID := broadcastNotificationIDPrefix + bn.id
+	n := notifications.Get(eventID)
+	if n != nil {
+		// Already active!
+		return ErrSkipAlreadyActive
+	}
+
+	// Prepare notification for displaying.
+	n = bn.Notification
+	n.EventID = eventID
+	n.GUID = ""
+	n.State = ""
+	n.SelectedActionID = ""
+
+	// It is okay to edit the notification, as they are loaded from the file every time.
+	// Add dismiss button if the notification is not permanent.
+	if !bn.Permanent {
+		n.AvailableActions = append(n.AvailableActions, &notifications.Action{
+			ID:   "ack",
+			Text: "Got it!",
+		})
+	}
+	n.SetActionFunction(markBroadcastAsRead)
+
+	// Display notification.
+	n.Save()
+
+	// Attach to module to raise more awareness.
+	if bn.AttachToModule {
+		n.AttachToModule(module)
+	}
+
+	return nil
+}
+
+func removeBroadcast(bn *BroadcastNotification, bss *BroadcastStates) (removed bool) {
+	// Remove any active notification.
+	n := notifications.Get(broadcastNotificationIDPrefix + bn.id)
+	if n != nil {
+		removed = true
+		n.Delete()
+	}
+
+	// Remove any state.
+	if bss != nil {
+		delete(bss.States, bn.id)
+	}
+
+	return
+}
+
+var savingBroadcastStateLock sync.Mutex
+
+func markBroadcastAsRead(ctx context.Context, n *notifications.Notification) error {
+	// Lock persisting broadcast state.
+	savingBroadcastStateLock.Lock()
+	defer savingBroadcastStateLock.Unlock()
+
+	// Get notification data.
+	var broadcastID, actionID string
+	func() {
+		n.Lock()
+		defer n.Unlock()
+		broadcastID = strings.TrimPrefix(n.EventID, broadcastNotificationIDPrefix)
+		actionID = n.SelectedActionID
+	}()
+
+	// Check response.
+	switch actionID {
+	case "ack":
+	case "":
+		return fmt.Errorf("no action ID for %s", broadcastID)
+	default:
+		return fmt.Errorf("unexpected action ID for %s: %s", broadcastID, actionID)
+	}
+
+	// Get broadcast notification states.
+	bss, err := getBroadcastStates()
+	if err != nil {
+		if !errors.Is(err, database.ErrNotFound) {
+			return fmt.Errorf("failed to get broadcast notifications states: %w", err)
+		}
+		bss = newBroadcastStates()
+	}
+
+	// Get state for this notification.
+	bs, ok := bss.States[broadcastID]
+	if !ok {
+		bs = &BroadcastState{}
+		bss.States[broadcastID] = bs
+	}
+
+	// Delete to allow for timely repeats.
+	n.Delete()
+
+	// Mark as read and save to DB.
+	log.Infof("broadcasts: user acknowledged broadcast %s", broadcastID)
+	bs.Read = time.Now()
+	return bss.save()
+}
diff --git a/broadcasts/state.go b/broadcasts/state.go
new file mode 100644
index 00000000..afe8994c
--- /dev/null
+++ b/broadcasts/state.go
@@ -0,0 +1,64 @@
+package broadcasts
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/safing/portbase/database/record"
+)
+
+const broadcastStatesDBKey = "core:broadcasts/state"
+
+// BroadcastStates holds states for broadcast notifications.
+type BroadcastStates struct {
+	record.Base
+	sync.Mutex
+
+	States map[string]*BroadcastState
+}
+
+// BroadcastState holds state for a single broadcast notifications.
+type BroadcastState struct {
+	Read time.Time
+}
+
+func (bss *BroadcastStates) save() error {
+	return db.Put(bss)
+}
+
+// getbroadcastStates returns the broadcast states from the database.
+func getBroadcastStates() (*BroadcastStates, error) {
+	r, err := db.Get(broadcastStatesDBKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unwrap.
+	if r.IsWrapped() {
+		// Only allocate a new struct, if we need it.
+		newRecord := &BroadcastStates{}
+		err = record.Unwrap(r, newRecord)
+		if err != nil {
+			return nil, err
+		}
+		return newRecord, nil
+	}
+
+	// or adjust type
+	newRecord, ok := r.(*BroadcastStates)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *BroadcastStates, but %T", r)
+	}
+	return newRecord, nil
+}
+
+// newBroadcastStates returns a new BroadcastStates.
+func newBroadcastStates() *BroadcastStates {
+	bss := &BroadcastStates{
+		States: make(map[string]*BroadcastState),
+	}
+	bss.SetKey(broadcastStatesDBKey)
+
+	return bss
+}
diff --git a/core/core.go b/core/core.go
index 80bd6343..8b70cba6 100644
--- a/core/core.go
+++ b/core/core.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/modules/subsystems"
+	_ "github.com/safing/portmaster/broadcasts"
 	_ "github.com/safing/portmaster/netenv"
 	_ "github.com/safing/portmaster/status"
 	_ "github.com/safing/portmaster/ui"
@@ -25,7 +26,7 @@ var (
 )
 
 func init() {
-	module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "interception", "compat")
+	module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "interception", "compat", "broadcasts")
 	subsystems.Register(
 		"core",
 		"Core",

From 92cc733aca318f45c8bef06b66be0cadb915039d Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 23 Jun 2022 16:30:55 +0200
Subject: [PATCH 73/86] Add broadcast notifications test data

---
 broadcasts/testdata/README.md          |  9 +++++++++
 broadcasts/testdata/notifications.yaml | 22 ++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 broadcasts/testdata/README.md
 create mode 100644 broadcasts/testdata/notifications.yaml

diff --git a/broadcasts/testdata/README.md b/broadcasts/testdata/README.md
new file mode 100644
index 00000000..1752a7f6
--- /dev/null
+++ b/broadcasts/testdata/README.md
@@ -0,0 +1,9 @@
+# Testing Broadcast Notifications
+
+```
+# Reset state
+curl -X POST http://127.0.0.1:817/api/v1/broadcasts/reset-state
+
+# Simulate notifications
+curl --upload-file notifications.yaml http://127.0.0.1:817/api/v1/broadcasts/simulate
+```
diff --git a/broadcasts/testdata/notifications.yaml b/broadcasts/testdata/notifications.yaml
new file mode 100644
index 00000000..6c60d6b5
--- /dev/null
+++ b/broadcasts/testdata/notifications.yaml
@@ -0,0 +1,22 @@
+notifications:
+  test1:
+    title: "[TEST] Normal Broadcast"
+    message: "This is a normal broadcast without matching. (#1)"
+  test2:
+    title: "[TEST] Permanent Broadcast"
+    message: "This is a permanent broadcast without matching. (#2)"
+    type: 1 # Warning
+    permanent: true
+  test3:
+    title: "[TEST] Repeating Broadcast"
+    message: "This is a repeating broadcast without matching. (#3)"
+    repeat: "1m"
+  test4:
+    title: "[TEST] Matching Broadcast: PM version"
+    message: "This is a normal broadcast that matches the PM version. (#4)"
+    match: "NumericVersion > 8000"
+  test5:
+    title: "[TEST] Important Update"
+    message: "A criticial update has been released, please update immediately. (#5)"
+    type: 3 # Error
+    attachToModule: true

From 492afc6dcfa2882749cc5ecbe5ddff19c676c612 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 23 Jun 2022 16:31:04 +0200
Subject: [PATCH 74/86] Update dependencies

---
 go.mod | 25 ++++++++++++-------------
 go.sum | 45 ++++++++++++++++++++++++++++-----------------
 2 files changed, 40 insertions(+), 30 deletions(-)

diff --git a/go.mod b/go.mod
index 997bdcb0..ab267425 100644
--- a/go.mod
+++ b/go.mod
@@ -7,23 +7,24 @@ require (
 	github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d
 	github.com/coreos/go-iptables v0.6.0
 	github.com/florianl/go-nfqueue v1.3.1
+	github.com/ghodss/yaml v1.0.0
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/go-version v1.4.0
-	github.com/miekg/dns v1.1.49
+	github.com/hashicorp/go-version v1.5.0
+	github.com/miekg/dns v1.1.50
 	github.com/oschwald/maxminddb-golang v1.9.0
-	github.com/safing/portbase v0.14.4
-	github.com/safing/spn v0.4.11
+	github.com/safing/portbase v0.14.5
+	github.com/safing/spn v0.4.12
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/spf13/cobra v1.4.0
+	github.com/spf13/cobra v1.5.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220513224357-95641704303c
-	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
-	golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a
+	golang.org/x/net v0.0.0-20220622184535-263ec571b305
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664
 )
 
 require (
@@ -35,7 +36,6 @@ require (
 	github.com/bluele/gcache v0.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
@@ -71,10 +71,9 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/tools v0.1.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
diff --git a/go.sum b/go.sum
index 7fe99705..40b1435d 100644
--- a/go.sum
+++ b/go.sum
@@ -250,6 +250,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -530,8 +531,9 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
 github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E=
+github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -688,8 +690,9 @@ github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7Xn
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
-github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
+github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -818,8 +821,9 @@ github.com/safing/portbase v0.14.0/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X
 github.com/safing/portbase v0.14.1/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
-github.com/safing/portbase v0.14.4 h1:yEWlvh2w0gVdjrA9oo5QPFV5oluJ6YDoKRPtmFb9B4c=
 github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc=
+github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=
@@ -837,6 +841,7 @@ github.com/safing/portmaster v0.8.5/go.mod h1:MqOlFwHcIx/109Ugutz/CG23znuuXCRVHc
 github.com/safing/portmaster v0.8.7/go.mod h1:RUgCWt5v22jDUOtJfOwApi//Kt8RTZQhlREcBc+L4z8=
 github.com/safing/portmaster v0.8.9-interdep/go.mod h1:1hK7QpvFVlb/sglkc3SKj+RXMGBuk0wqO2s3pvMg1Xs=
 github.com/safing/portmaster v0.8.9/go.mod h1:tv0rxO76hrpBLdArN7YTypOaseH6zgQ2gLI2zCknk9Q=
+github.com/safing/portmaster v0.8.14-interdep/go.mod h1:HIkaE8wCXr8ULyZSWFkQNNY9obpMufxizXZugnjHLK0=
 github.com/safing/spn v0.3.4/go.mod h1:TfzNsZCbnlWv0UFDILFOUSudVKJZlnBVoR1fDXrjOK0=
 github.com/safing/spn v0.3.5/go.mod h1:jHkFF2Yu1fnjFu4KXjVA+iagMr/z4eB4p3jiwikvKj8=
 github.com/safing/spn v0.3.6/go.mod h1:RSeFb/h5Wt3yDVezXj3lhXJ/Iwd7FbtsGf5E+p5J2YQ=
@@ -854,14 +859,10 @@ github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
 github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
-github.com/safing/spn v0.4.8 h1:C5QOl03hypDaC4qvFtWjAKmSJIOjfqYVGT+sTnUIO/E=
-github.com/safing/spn v0.4.8/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
-github.com/safing/spn v0.4.9 h1:p+TRS2y1yAY4CYyHiVuy+V3NcA9p826NtYksuBaSl64=
-github.com/safing/spn v0.4.9/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
-github.com/safing/spn v0.4.10 h1:nZuWPBn1z0aBUDnnpG9pKZCmPAH2hgNE4cqiSB7JGU8=
-github.com/safing/spn v0.4.10/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.11 h1:Er3ZtBCqKaf0G6lmRKpm1LenlrWeaWUp8612bpeG+aM=
 github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
+github.com/safing/spn v0.4.12 h1:Tw7TUZEZR4yZy7L+ICRCketDk5L5x0s0pvrSUHFaKs4=
+github.com/safing/spn v0.4.12/go.mod h1:AUNgBrRwCcspC98ljptDnrPuHLn/BHSG+rSprV/5Wlc=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -905,8 +906,9 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
-github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
 github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -1094,8 +1096,10 @@ golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c=
 golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1133,8 +1137,9 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1215,8 +1220,10 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220513224357-95641704303c h1:nF9mHSvoKBLkQNQhJZNsc66z2UzAMUbLGjC95CF3pU0=
 golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220621193019-9d032be2e588/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220622184535-263ec571b305 h1:dAgbJ2SP4jD6XYfMNLVj0BF21jo2PjChrtGaAvF5M3I=
+golang.org/x/net v0.0.0-20220622184535-263ec571b305/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1248,8 +1255,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1374,8 +1382,11 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXHCguynGhKjWVsIUt5cY=
 golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 h1:wEZYwx+kK+KlZ0hpvP2Ls1Xr4+RWnlzGFwPP0aiDjIU=
+golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1471,13 +1482,13 @@ golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
-golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
+golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=

From 1dfab49024137381ee288cd27279d63beecbe94b Mon Sep 17 00:00:00 2001
From: Dave Gson <3080765+davegson@users.noreply.github.com>
Date: Tue, 28 Jun 2022 11:49:29 +0000
Subject: [PATCH 75/86] Update Primary Tag-Line

brings it up to speed with the website
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 621b7007..92d99b2e 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Take Back Control of Your Computer
+# Control Your Computer,<br>[Get Peace of Mind](https://safing.io/portmaster/)
 
 Portmaster is a free and open-source application that puts you back in charge over all your computer's network connections.
 Developed in the EU 🇪🇺, Austria.

From d48d511dfe78e1892a80beb62f9acdf8ba4cf6c8 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 30 Jun 2022 11:17:01 +0200
Subject: [PATCH 76/86] Create dependabot.yml

---
 .github/dependabot.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..f6150ead
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"

From f57facc374d5634d27a3f9c37304058314c1c2a7 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 30 Jun 2022 11:17:23 +0200
Subject: [PATCH 77/86] Create codeql-analysis.yml

---
 .github/workflows/codeql-analysis.yml | 72 +++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 .github/workflows/codeql-analysis.yml

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 00000000..fd52cfbc
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "develop", master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "develop" ]
+  schedule:
+    - cron: '43 14 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

From 44b54067d3d50d27bfec77b735ea65ce05250833 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 30 Jun 2022 09:18:02 +0000
Subject: [PATCH 78/86] Bump github.com/hashicorp/go-version from 1.5.0 to
 1.6.0

Bumps [github.com/hashicorp/go-version](https://github.com/hashicorp/go-version) from 1.5.0 to 1.6.0.
- [Release notes](https://github.com/hashicorp/go-version/releases)
- [Changelog](https://github.com/hashicorp/go-version/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hashicorp/go-version/compare/v1.5.0...v1.6.0)

---
updated-dependencies:
- dependency-name: github.com/hashicorp/go-version
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index ab267425..6561634a 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/go-version v1.5.0
+	github.com/hashicorp/go-version v1.6.0
 	github.com/miekg/dns v1.1.50
 	github.com/oschwald/maxminddb-golang v1.9.0
 	github.com/safing/portbase v0.14.5
diff --git a/go.sum b/go.sum
index 40b1435d..ad502342 100644
--- a/go.sum
+++ b/go.sum
@@ -532,8 +532,9 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E=
 github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -859,7 +860,6 @@ github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
 github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
-github.com/safing/spn v0.4.11 h1:Er3ZtBCqKaf0G6lmRKpm1LenlrWeaWUp8612bpeG+aM=
 github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.12 h1:Tw7TUZEZR4yZy7L+ICRCketDk5L5x0s0pvrSUHFaKs4=
 github.com/safing/spn v0.4.12/go.mod h1:AUNgBrRwCcspC98ljptDnrPuHLn/BHSG+rSprV/5Wlc=

From 20b9bdc74347bf88d59c44fb2b80a33f5a726db1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 30 Jun 2022 09:18:11 +0000
Subject: [PATCH 79/86] Bump github.com/stretchr/testify from 1.7.1 to 1.8.0

Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.7.1 to 1.8.0.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.7.1...v1.8.0)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 4 ++--
 go.sum | 8 +++++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/go.mod b/go.mod
index ab267425..25ac29e3 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/safing/spn v0.4.12
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
@@ -75,5 +75,5 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/tools v0.1.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 40b1435d..c052dbcf 100644
--- a/go.sum
+++ b/go.sum
@@ -859,7 +859,6 @@ github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
 github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
-github.com/safing/spn v0.4.11 h1:Er3ZtBCqKaf0G6lmRKpm1LenlrWeaWUp8612bpeG+aM=
 github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.12 h1:Tw7TUZEZR4yZy7L+ICRCketDk5L5x0s0pvrSUHFaKs4=
 github.com/safing/spn v0.4.12/go.mod h1:AUNgBrRwCcspC98ljptDnrPuHLn/BHSG+rSprV/5Wlc=
@@ -924,6 +923,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -931,8 +931,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tannerryan/ring v1.1.2 h1:iXayOjqHQOLzuy9GwSKuG3nhWfzQkldMlQivcgIr7gQ=
 github.com/tannerryan/ring v1.1.2/go.mod h1:DkELJEjbZhJBtFKR9Xziwj3HKZnb/knRgljNqp65vH4=
@@ -1684,8 +1685,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=

From d93c3f60ce3f6e122661e0f95d7d95e16dabcfd9 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 12 Jul 2022 11:27:46 +0200
Subject: [PATCH 80/86] Add additional field to internal SQL model

---
 netquery/database.go | 7 +++++++
 netquery/manager.go  | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/netquery/database.go b/netquery/database.go
index 5197868d..57aa6f6c 100644
--- a/netquery/database.go
+++ b/netquery/database.go
@@ -92,6 +92,13 @@ type (
 		Allowed         *bool             `sqlite:"allowed"`
 		ProfileRevision int               `sqlite:"profile_revision"`
 		ExitNode        *string           `sqlite:"exit_node"`
+
+		// FIXME(ppacher): support "NOT" in search query to get rid of the following helper fields
+		SPNUsed bool `sqlite:"spn_used"` // could use "exit_node IS NOT NULL" or "exit IS NULL"
+		Active  bool `sqlite:"active"`   // could use "ended IS NOT NULL" or "ended IS NULL"
+
+		// FIXME(ppacher): we need to profile here for "suggestion" support. It would be better to keep a table of profiles in sqlite and use joins here
+		ProfileName string `sqlite:"profile_name"`
 	}
 )
 
diff --git a/netquery/manager.go b/netquery/manager.go
index 2620e510..d5d4992f 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -177,6 +177,7 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		ProfileID:       conn.ProcessContext.Source + "/" + conn.ProcessContext.Profile,
 		Path:            conn.ProcessContext.BinaryPath,
 		ProfileRevision: int(conn.ProfileRevisionCounter),
+		ProfileName:     conn.ProcessContext.ProfileName,
 	}
 
 	switch conn.Type {
@@ -200,6 +201,9 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 	if conn.Ended > 0 {
 		ended := time.Unix(conn.Ended, 0)
 		c.Ended = &ended
+		c.Active = false
+	} else {
+		c.Active = true
 	}
 
 	extraData := map[string]interface{}{
@@ -210,6 +214,7 @@ func convertConnection(conn *network.Connection) (*Conn, error) {
 		extraData["tunnel"] = conn.TunnelContext
 		exitNode := conn.TunnelContext.GetExitNodeID()
 		c.ExitNode = &exitNode
+		c.SPNUsed = true
 	}
 
 	if conn.DNSContext != nil {

From c9a217a9b89655ab95ccfe76ccebabbf6e5ec884 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 14 Jul 2022 13:44:08 +0200
Subject: [PATCH 81/86] Fix go.mod

---
 go.mod |  89 ++++++++++++++++++++------
 go.sum | 193 +++++++++++++++------------------------------------------
 2 files changed, 121 insertions(+), 161 deletions(-)

diff --git a/go.mod b/go.mod
index 3100d1f9..9fd7c7b5 100644
--- a/go.mod
+++ b/go.mod
@@ -1,34 +1,87 @@
 module github.com/safing/portmaster
 
-go 1.15
+go 1.18
 
 require (
 	github.com/agext/levenshtein v1.2.3
 	github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d
 	github.com/coreos/go-iptables v0.6.0
-	github.com/florianl/go-nfqueue v1.3.0
+	github.com/florianl/go-nfqueue v1.3.1
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/go-version v1.4.0
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/mdlayher/socket v0.2.2 // indirect
-	github.com/miekg/dns v1.1.46
-	github.com/oschwald/maxminddb-golang v1.8.0
-	github.com/safing/portbase v0.14.0
-	github.com/safing/spn v0.4.3
+	github.com/hashicorp/go-version v1.6.0
+	github.com/miekg/dns v1.1.50
+	github.com/oschwald/maxminddb-golang v1.9.0
+	github.com/safing/portbase v0.14.5
+	github.com/safing/spn v0.4.12
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/spf13/cobra v1.3.0
-	github.com/stretchr/testify v1.7.0
+	github.com/spf13/cobra v1.5.0
+	github.com/stretchr/testify v1.7.1
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
-	zombiezen.com/go/sqlite v0.9.2
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e
+	zombiezen.com/go/sqlite v0.10.0
 )
 
-replace github.com/safing/spn => ../spn
-
-replace github.com/safing/portbase => ../portbase
+require (
+	github.com/VictoriaMetrics/metrics v1.18.1 // indirect
+	github.com/aead/ecdh v0.2.0 // indirect
+	github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 // indirect
+	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/native v1.0.0 // indirect
+	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mdlayher/netlink v1.6.0 // indirect
+	github.com/mdlayher/socket v0.2.3 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/rot256/pblind v0.0.0-20211117203330-22455f90b565 // indirect
+	github.com/safing/jess v0.2.3 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/seehuhn/fortuna v1.0.1 // indirect
+	github.com/seehuhn/sha256d v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.14.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.4 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.5.0 // indirect
+	github.com/valyala/fastrand v1.1.0 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/tools v0.1.11 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	modernc.org/libc v1.16.17 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.1.1 // indirect
+	modernc.org/sqlite v1.17.3 // indirect
+)
diff --git a/go.sum b/go.sum
index 390453ed..1790fa2a 100644
--- a/go.sum
+++ b/go.sum
@@ -251,6 +251,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -475,7 +476,6 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -533,8 +533,10 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
 github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -651,7 +653,7 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
 github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
@@ -693,8 +695,9 @@ github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7Xn
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
-github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
+github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -825,8 +828,9 @@ github.com/safing/portbase v0.14.0/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X
 github.com/safing/portbase v0.14.1/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
-github.com/safing/portbase v0.14.4 h1:yEWlvh2w0gVdjrA9oo5QPFV5oluJ6YDoKRPtmFb9B4c=
 github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
+github.com/safing/portbase v0.14.5 h1:+8H+mQ7AFjA04M7UPq0490pj3/+nvJj3pEUP1PYTMYc=
+github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
 github.com/safing/portmaster v0.7.3/go.mod h1:o//kZ8eE+5vT1V22mgnxHIAdlEz42sArsK5OF2Lf/+s=
 github.com/safing/portmaster v0.7.4/go.mod h1:Q93BWdF1oAL0oUMukshl8W1aPZhmrlTGi6tFTFc3pTw=
 github.com/safing/portmaster v0.7.6/go.mod h1:qOs9hQtvAzTVICRbwLg3vddqOaqJHeWBjWQ0C+TJ/Bw=
@@ -844,6 +848,7 @@ github.com/safing/portmaster v0.8.5/go.mod h1:MqOlFwHcIx/109Ugutz/CG23znuuXCRVHc
 github.com/safing/portmaster v0.8.7/go.mod h1:RUgCWt5v22jDUOtJfOwApi//Kt8RTZQhlREcBc+L4z8=
 github.com/safing/portmaster v0.8.9-interdep/go.mod h1:1hK7QpvFVlb/sglkc3SKj+RXMGBuk0wqO2s3pvMg1Xs=
 github.com/safing/portmaster v0.8.9/go.mod h1:tv0rxO76hrpBLdArN7YTypOaseH6zgQ2gLI2zCknk9Q=
+github.com/safing/portmaster v0.8.14-interdep/go.mod h1:HIkaE8wCXr8ULyZSWFkQNNY9obpMufxizXZugnjHLK0=
 github.com/safing/spn v0.3.4/go.mod h1:TfzNsZCbnlWv0UFDILFOUSudVKJZlnBVoR1fDXrjOK0=
 github.com/safing/spn v0.3.5/go.mod h1:jHkFF2Yu1fnjFu4KXjVA+iagMr/z4eB4p3jiwikvKj8=
 github.com/safing/spn v0.3.6/go.mod h1:RSeFb/h5Wt3yDVezXj3lhXJ/Iwd7FbtsGf5E+p5J2YQ=
@@ -861,14 +866,9 @@ github.com/safing/spn v0.4.3/go.mod h1:YHtg3FkZviN8T7db4BdRffbYO1pO7w9SydQatLmvW
 github.com/safing/spn v0.4.5/go.mod h1:mkQA5pYM1SUd4JkTyuwXFycFMGQXLTd9RUJuY2vqccM=
 github.com/safing/spn v0.4.6/go.mod h1:AmZ+rore+6DQp0GSchIAXPn8ij0Knyw7uy4PbMLljXg=
 github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3hV8=
-github.com/safing/spn v0.4.8 h1:C5QOl03hypDaC4qvFtWjAKmSJIOjfqYVGT+sTnUIO/E=
-github.com/safing/spn v0.4.8/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
-github.com/safing/spn v0.4.9 h1:p+TRS2y1yAY4CYyHiVuy+V3NcA9p826NtYksuBaSl64=
-github.com/safing/spn v0.4.9/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
-github.com/safing/spn v0.4.10 h1:nZuWPBn1z0aBUDnnpG9pKZCmPAH2hgNE4cqiSB7JGU8=
-github.com/safing/spn v0.4.10/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
-github.com/safing/spn v0.4.11 h1:Er3ZtBCqKaf0G6lmRKpm1LenlrWeaWUp8612bpeG+aM=
 github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
+github.com/safing/spn v0.4.12 h1:Tw7TUZEZR4yZy7L+ICRCketDk5L5x0s0pvrSUHFaKs4=
+github.com/safing/spn v0.4.12/go.mod h1:AUNgBrRwCcspC98ljptDnrPuHLn/BHSG+rSprV/5Wlc=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -912,8 +912,9 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
-github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
 github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -976,7 +977,6 @@ github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp1
 github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
 github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
 github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
-github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
 github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
@@ -1090,7 +1090,6 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -1103,8 +1102,10 @@ golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c=
 golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1140,11 +1141,11 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1225,8 +1226,10 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220513224357-95641704303c h1:nF9mHSvoKBLkQNQhJZNsc66z2UzAMUbLGjC95CF3pU0=
 golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220621193019-9d032be2e588/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1258,8 +1261,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1324,7 +1328,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1361,7 +1364,6 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1386,8 +1388,11 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXHCguynGhKjWVsIUt5cY=
 golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1479,19 +1484,18 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
-golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
+golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -1711,128 +1715,31 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
-modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
-modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
-modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag=
-modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw=
-modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ=
-modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c=
-modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo=
-modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg=
-modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I=
-modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs=
-modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8=
-modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE=
-modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk=
-modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w=
-modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE=
-modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8=
-modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc=
-modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU=
-modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE=
-modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk=
-modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI=
-modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE=
-modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg=
-modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74=
-modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU=
-modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
-modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
-modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
-modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
-modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
-modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
-modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY=
-modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
-modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU=
-modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko=
-modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA=
-modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4=
-modernc.org/ccgo/v3 v3.14.0/go.mod h1:hBrkiBlUwvr5vV/ZH9YzXIp982jKE8Ek8tR1ytoAL6Q=
-modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
-modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
-modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8=
-modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I=
-modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
+modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
+modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
 modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
-modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
-modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q=
-modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg=
-modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M=
-modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU=
-modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE=
-modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso=
-modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8=
-modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8=
-modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I=
-modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk=
-modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY=
-modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE=
-modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg=
-modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM=
-modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg=
-modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo=
-modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8=
-modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ=
-modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA=
-modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM=
-modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg=
-modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE=
-modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM=
-modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU=
-modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
-modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
-modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
-modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
-modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
-modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
-modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
-modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE=
-modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY=
-modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ=
-modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
-modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
-modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
-modernc.org/libc v1.13.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
-modernc.org/libc v1.13.2/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
-modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
-modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34=
-modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ=
-modernc.org/libc v1.14.5 h1:DAHvwGoVRDZs5iJXnX9RJrgXSsorupCWmJ2ac964Owk=
-modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak=
-modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
+modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A=
+modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU=
+modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
+modernc.org/libc v1.16.17 h1:rXo8IZJvP+QSN1KrlV23dtkM3XfGYXjx3RbLLzBtndM=
+modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
 modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
 modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
-modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
-modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
+modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU=
+modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
 modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.14.5 h1:bYrrjwH9Y7QUGk1MbchZDhRfmpGuEAs/D45sVjNbfvs=
-modernc.org/sqlite v1.14.5/go.mod h1:YyX5Rx0WbXokitdWl2GJIDy4BrPxBP0PwwhpXOHCDLE=
+modernc.org/sqlite v1.17.3 h1:iE+coC5g17LtByDYDWKpR6m2Z9022YrSh3bumwOnIrI=
+modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k=
 modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
-modernc.org/tcl v1.10.0/go.mod h1:WzWapmP/7dHVhFoyPpEaNSVTL8xtewhouN/cqSJ5A2s=
+modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
 modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.2.21/go.mod h1:uXrObx4pGqXWIMliC5MiKuwAyMrltzwpteOFUP1PWCc=
-modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g=
+modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
 nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
@@ -1842,5 +1749,5 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-zombiezen.com/go/sqlite v0.9.2 h1:bgY6e0BksSrrQqM9gK+HYLbMj1zxcRWHijJBlYnRpCw=
-zombiezen.com/go/sqlite v0.9.2/go.mod h1:M/gb7zbJfWDAUQAsw/9wf0c3P1cHt6Mv9zrtVmAs13Y=
+zombiezen.com/go/sqlite v0.10.0 h1:hegW0Y8c/fSJ2VjbjBeiKJaQOISNr4EUTx1VZx94Q9Y=
+zombiezen.com/go/sqlite v0.10.0/go.mod h1:tOd9u3peffVYnXOedepSJmX92n/mbqf594wcJ+29jf8=

From 51a2d27522d690f6c7cb45de7ec075ef35b1df9d Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Thu, 14 Jul 2022 13:44:31 +0200
Subject: [PATCH 82/86] Implement some review suggestions

---
 netquery/manager.go    | 14 +------------
 netquery/module_api.go | 46 +++++++++++-------------------------------
 2 files changed, 13 insertions(+), 47 deletions(-)

diff --git a/netquery/manager.go b/netquery/manager.go
index d5d4992f..b6649483 100644
--- a/netquery/manager.go
+++ b/netquery/manager.go
@@ -78,12 +78,6 @@ func (mng *Manager) runtimeGet(keyOrPrefix string) ([]record.Record, error) {
 // HandleFeed handles and persists updates one after each other! Depending on the system
 // load the user might want to use a buffered channel for feed.
 func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connection) {
-	// count the number of inserted rows for logging purposes.
-	//
-	// TODO(ppacher): how to handle the, though unlikely case, of a
-	// overflow to 0 here?
-	var count uint64
-
 	for {
 		select {
 		case <-ctx.Done():
@@ -101,7 +95,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 				continue
 			}
 
-			log.Infof("netquery: persisting create/update to connection %s", conn.ID)
+			log.Tracef("netquery: updating connection %s", conn.ID)
 
 			if err := mng.store.Save(ctx, *model); err != nil {
 				log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err)
@@ -118,12 +112,6 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect
 			if err := mng.pushConnUpdate(ctx, *cloned, *model); err != nil {
 				log.Errorf("netquery: failed to push update for conn %s via database system: %s", conn.ID, err)
 			}
-
-			count++
-
-			if count%20 == 0 {
-				log.Debugf("netquery: persisted %d connections so far", count)
-			}
 		}
 	}
 }
diff --git a/netquery/module_api.go b/netquery/module_api.go
index dd15189e..e5127b90 100644
--- a/netquery/module_api.go
+++ b/netquery/module_api.go
@@ -41,9 +41,8 @@ func (m *Module) Prepare() error {
 	var err error
 
 	m.db = database.NewInterface(&database.Options{
-		Local:     true,
-		Internal:  true,
-		CacheSize: 0,
+		Local:    true,
+		Internal: true,
 	})
 
 	m.sqlStore, err = NewInMemory()
@@ -71,12 +70,12 @@ func (m *Module) Prepare() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:        "netquery/query",
 		MimeType:    "application/json",
-		Read:        api.PermitAnyone,
-		Write:       api.PermitAnyone,
+		Read:        api.PermitUser,
+		Write:       api.PermitUser,
 		BelongsTo:   m.Module,
 		HandlerFunc: queryHander.ServeHTTP,
-		Name:        "Query In-Memory Database",
-		Description: "Query the in-memory sqlite database",
+		Name:        "Query Connections",
+		Description: "Query the in-memory sqlite connection database.",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
@@ -84,12 +83,12 @@ func (m *Module) Prepare() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:        "netquery/charts/connection-active",
 		MimeType:    "application/json",
-		Read:        api.PermitAnyone,
-		Write:       api.PermitAnyone,
+		Read:        api.PermitUser,
+		Write:       api.PermitUser,
 		BelongsTo:   m.Module,
 		HandlerFunc: chartHandler.ServeHTTP,
-		Name:        "Query In-Memory Database",
-		Description: "Query the in-memory sqlite database",
+		Name:        "Active Connections Chart",
+		Description: "Query the in-memory sqlite connection database and return a chart of active connections.",
 	}); err != nil {
 		return fmt.Errorf("failed to register API endpoint: %w", err)
 	}
@@ -142,36 +141,15 @@ func (mod *Module) Start() error {
 				if err != nil {
 					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
 				} else {
-					log.Infof("netquery: successfully removed %d old rows that ended before %s", count, threshold)
+					log.Tracef("netquery: successfully removed %d old rows that ended before %s", count, threshold)
 				}
 			}
 		}
 	})
 
-	mod.StartWorker("netquery-row-counter", func(ctx context.Context) error {
-		for {
-			select {
-			case <-ctx.Done():
-				return nil
-			case <-time.After(1 * time.Second):
-				count, err := mod.sqlStore.CountRows(ctx)
-				if err != nil {
-					log.Errorf("netquery: failed to count number of rows in memory: %s", err)
-				} else {
-					log.Infof("netquery: currently holding %d rows in memory", count)
-				}
-
-				/*
-					if err := sqlStore.dumpTo(ctx, os.Stderr); err != nil {
-						log.Errorf("netquery: failed to dump sqlite memory content: %w", err)
-					}
-				*/
-			}
-		}
-	})
-
 	// for debugging, we provide a simple direct SQL query interface using
 	// the runtime database
+	// FIXME: Expose only in dev mode.
 	_, err := NewRuntimeQueryRunner(mod.sqlStore, "netquery/query/", runtime.DefaultRegistry)
 	if err != nil {
 		return fmt.Errorf("failed to set up runtime SQL query runner: %w", err)

From 71bedc63613cdc301b29c6e96458c2b4b5c7ef45 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 19 Jul 2022 15:31:35 +0200
Subject: [PATCH 83/86] Automatically append arguments required by Electron to
 run on wayland

---
 cmds/portmaster-start/run.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/cmds/portmaster-start/run.go b/cmds/portmaster-start/run.go
index 8881b2d6..046549d0 100644
--- a/cmds/portmaster-start/run.go
+++ b/cmds/portmaster-start/run.go
@@ -121,6 +121,20 @@ func getExecArgs(opts *Options, cmdArgs []string) []string {
 	if stdinSignals {
 		args = append(args, "--input-signals")
 	}
+
+	// see https://www.freedesktop.org/software/systemd/man/pam_systemd.html#type=
+	if xdgSessionType := os.Getenv("XDG_SESSION_TYPE"); xdgSessionType == "wayland" && opts.Identifier == "app/portmaster-app.zip" {
+		// we're running the Portmaster UI App under Wayland so make sure we add some arguments
+		// required by Electron
+		args = append(args,
+			[]string{
+				"--enable-features=UseOzonePlatform",
+				"--ozone-platform=wayland",
+			}...,
+		)
+
+	}
+
 	args = append(args, cmdArgs...)
 	return args
 }

From f4030d579bb19c4fa3efeee96423b2556abfc188 Mon Sep 17 00:00:00 2001
From: Patrick Pacher <patrick.pacher@gmail.com>
Date: Tue, 19 Jul 2022 15:58:08 +0200
Subject: [PATCH 84/86] Fix linter error and make identifier check more
 explicit

---
 cmds/portmaster-start/run.go | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/cmds/portmaster-start/run.go b/cmds/portmaster-start/run.go
index 046549d0..97dd38f7 100644
--- a/cmds/portmaster-start/run.go
+++ b/cmds/portmaster-start/run.go
@@ -122,17 +122,18 @@ func getExecArgs(opts *Options, cmdArgs []string) []string {
 		args = append(args, "--input-signals")
 	}
 
-	// see https://www.freedesktop.org/software/systemd/man/pam_systemd.html#type=
-	if xdgSessionType := os.Getenv("XDG_SESSION_TYPE"); xdgSessionType == "wayland" && opts.Identifier == "app/portmaster-app.zip" {
-		// we're running the Portmaster UI App under Wayland so make sure we add some arguments
-		// required by Electron
-		args = append(args,
-			[]string{
-				"--enable-features=UseOzonePlatform",
-				"--ozone-platform=wayland",
-			}...,
-		)
-
+	if opts.Identifier == "app/portmaster-app.zip" {
+		// see https://www.freedesktop.org/software/systemd/man/pam_systemd.html#type=
+		if xdgSessionType := os.Getenv("XDG_SESSION_TYPE"); xdgSessionType == "wayland" {
+			// we're running the Portmaster UI App under Wayland so make sure we add some arguments
+			// required by Electron
+			args = append(args,
+				[]string{
+					"--enable-features=UseOzonePlatform",
+					"--ozone-platform=wayland",
+				}...,
+			)
+		}
 	}
 
 	args = append(args, cmdArgs...)

From a0e036e263e415a15c56f5f7f46edc74d79ecee6 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 19 Jul 2022 17:05:47 +0200
Subject: [PATCH 85/86] Update deps

---
 go.mod | 17 +++++------------
 go.sum | 10 ++++++++--
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/go.mod b/go.mod
index 95a6a374..45638ee2 100644
--- a/go.mod
+++ b/go.mod
@@ -15,24 +15,17 @@ require (
 	github.com/miekg/dns v1.1.50
 	github.com/oschwald/maxminddb-golang v1.9.0
 	github.com/safing/portbase v0.14.5
-	github.com/safing/spn v0.4.12
+	github.com/safing/spn v0.4.13
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	github.com/tannerryan/ring v1.1.2
 	github.com/tevino/abool v1.2.0
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
 	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e
-	zombiezen.com/go/sqlite v0.10.0
-	github.com/stretchr/testify v1.8.0
-	github.com/tannerryan/ring v1.1.2
-	github.com/tevino/abool v1.2.0
-	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
-	golang.org/x/net v0.0.0-20220622184535-263ec571b305
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8
+	zombiezen.com/go/sqlite v0.10.1
 )
 
 require (
@@ -86,7 +79,7 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/tools v0.1.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.16.17 // indirect
 	modernc.org/mathutil v1.4.1 // indirect
 	modernc.org/memory v1.1.1 // indirect
diff --git a/go.sum b/go.sum
index 168ff329..0b6989af 100644
--- a/go.sum
+++ b/go.sum
@@ -869,6 +869,8 @@ github.com/safing/spn v0.4.7/go.mod h1:NoSG9K0OK9hrPC76yqWFS6RtvbqZdIc/KGOsC4T3h
 github.com/safing/spn v0.4.11/go.mod h1:nro/I6b2JnafeeqoMsQRqf6TaQeL9uLLZkUREtxLVDE=
 github.com/safing/spn v0.4.12 h1:Tw7TUZEZR4yZy7L+ICRCketDk5L5x0s0pvrSUHFaKs4=
 github.com/safing/spn v0.4.12/go.mod h1:AUNgBrRwCcspC98ljptDnrPuHLn/BHSG+rSprV/5Wlc=
+github.com/safing/spn v0.4.13 h1:5NXWUl/2EWyotrQhW3tD+3DYw7hEqQk0n0lHa+w4eFo=
+github.com/safing/spn v0.4.13/go.mod h1:rBeimIc1FHQOhX7lTh/LaFGRotmnwZIDWUSsPyeIDog=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -1230,6 +1232,8 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220621193019-9d032be2e588/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1391,6 +1395,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1748,5 +1754,5 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-zombiezen.com/go/sqlite v0.10.0 h1:hegW0Y8c/fSJ2VjbjBeiKJaQOISNr4EUTx1VZx94Q9Y=
-zombiezen.com/go/sqlite v0.10.0/go.mod h1:tOd9u3peffVYnXOedepSJmX92n/mbqf594wcJ+29jf8=
+zombiezen.com/go/sqlite v0.10.1 h1:PSgVSHeIVOGKbX7ZIQNXGKn3wcqM6JBnT4yS1OLjWbM=
+zombiezen.com/go/sqlite v0.10.1/go.mod h1:tOd9u3peffVYnXOedepSJmX92n/mbqf594wcJ+29jf8=

From ce176de2b4334e621b6252ff1adefacc0b5c0963 Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Tue, 19 Jul 2022 17:07:03 +0200
Subject: [PATCH 86/86] Bump versions

---
 cmds/portmaster-core/main.go  | 2 +-
 cmds/portmaster-start/main.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmds/portmaster-core/main.go b/cmds/portmaster-core/main.go
index b7f1d1ec..f3c5e0fd 100644
--- a/cmds/portmaster-core/main.go
+++ b/cmds/portmaster-core/main.go
@@ -19,7 +19,7 @@ import ( //nolint:gci,nolintlint
 
 func main() {
 	// set information
-	info.Set("Portmaster", "0.8.13", "AGPLv3", true)
+	info.Set("Portmaster", "0.9.0", "AGPLv3", true)
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("portmaster")
diff --git a/cmds/portmaster-start/main.go b/cmds/portmaster-start/main.go
index 6afc831d..d9f53816 100644
--- a/cmds/portmaster-start/main.go
+++ b/cmds/portmaster-start/main.go
@@ -77,7 +77,7 @@ func main() {
 	cobra.OnInitialize(initCobra)
 
 	// set meta info
-	info.Set("Portmaster Start", "0.8.8", "AGPLv3", false)
+	info.Set("Portmaster Start", "0.9.0", "AGPLv3", false)
 
 	// catch interrupt for clean shutdown
 	signalCh := make(chan os.Signal, 2)