Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

service/compat/api.go

@@ -1,14 +1,13 @@
 package compat
 
 import (
-	"github.com/safing/portbase/api"
+	"github.com/safing/portmaster/base/api"
 )
 
 func registerAPIEndpoints() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:        "compat/self-check",
 		Read:        api.PermitUser,
-		BelongsTo:   module,
 		ActionFunc:  selfcheckViaAPI,
 		Name:        "Run Integration Self-Check",
 		Description: "Runs a couple integration self-checks in order to see if the system integration works.",

service/compat/debug_default.go

@@ -2,7 +2,7 @@
 
 package compat
 
-import "github.com/safing/portbase/utils/debug"
+import "github.com/safing/portmaster/base/utils/debug"
 
 // AddToDebugInfo adds compatibility data to the given debug.Info.
 func AddToDebugInfo(di *debug.Info) {

service/compat/debug_linux.go

@@ -3,7 +3,7 @@ package compat
 import (
 	"fmt"
 
-	"github.com/safing/portbase/utils/debug"
+	"github.com/safing/portmaster/base/utils/debug"
 )
 
 // AddToDebugInfo adds compatibility data to the given debug.Info.

service/compat/debug_windows.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/safing/portbase/utils/debug"
+	"github.com/safing/portmaster/base/utils/debug"
 )
 
 // AddToDebugInfo adds compatibility data to the given debug.Info.

service/compat/module.go

@@ -1,22 +1,50 @@
 package compat
 
 import (
-	"context"
 	"errors"
+	"sync/atomic"
 	"time"
 
 	"github.com/tevino/abool"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/netenv"
 	"github.com/safing/portmaster/service/resolver"
 )
 
-var (
-	module *modules.Module
+// Compat is the compatibility check module.
+type Compat struct {
+	mgr      *mgr.Manager
+	instance instance
 
-	selfcheckTask           *modules.Task
+	selfcheckWorkerMgr            *mgr.WorkerMgr
+	cleanNotifyThresholdWorkerMgr *mgr.WorkerMgr
+
+	states *mgr.StateMgr
+}
+
+// Manager returns the module manager.
+func (u *Compat) Manager() *mgr.Manager {
+	return u.mgr
+}
+
+// States returns the module state manager.
+func (u *Compat) States() *mgr.StateMgr {
+	return u.states
+}
+
+// Start starts the module.
+func (u *Compat) Start() error {
+	return start()
+}
+
+// Stop stops the module.
+func (u *Compat) Stop() error {
+	return stop()
+}
+
+var (
 	selfcheckTaskRetryAfter = 15 * time.Second
 
 	// selfCheckIsFailing holds whether or not the self-check is currently
@@ -38,8 +66,6 @@ var (
 const selfcheckFailThreshold = 10
 
 func init() {
-	module = modules.Register("compat", prep, start, stop, "base", "network", "interception", "netenv", "notifications")
-
 	// Workaround resolver integration.
 	// See resolver/compat.go for details.
 	resolver.CompatDNSCheckInternalDomainScope = DNSCheckInternalDomainScope
@@ -55,35 +81,26 @@ func start() error {
 	startNotify()
 
 	selfcheckNetworkChangedFlag.Refresh()
-	selfcheckTask = module.NewTask("compatibility self-check", selfcheckTaskFunc).
-		Repeat(5 * time.Minute).
-		MaxDelay(selfcheckTaskRetryAfter).
-		Schedule(time.Now().Add(selfcheckTaskRetryAfter))
+	module.selfcheckWorkerMgr.Repeat(5 * time.Minute).Delay(selfcheckTaskRetryAfter)
+	module.cleanNotifyThresholdWorkerMgr.Repeat(1 * time.Hour)
 
-	module.NewTask("clean notify thresholds", cleanNotifyThreshold).
-		Repeat(1 * time.Hour)
-
-	return module.RegisterEventHook(
-		netenv.ModuleName,
-		netenv.NetworkChangedEvent,
-		"trigger compat self-check",
-		func(_ context.Context, _ interface{}) error {
-			selfcheckTask.Schedule(time.Now().Add(selfcheckTaskRetryAfter))
-			return nil
-		},
-	)
+	module.instance.NetEnv().EventNetworkChange.AddCallback("trigger compat self-check", func(_ *mgr.WorkerCtx, _ struct{}) (bool, error) {
+		module.selfcheckWorkerMgr.Delay(selfcheckTaskRetryAfter)
+		return false, nil
+	})
+	return nil
 }
 
 func stop() error {
-	selfcheckTask.Cancel()
-	selfcheckTask = nil
+	// selfcheckTask.Cancel()
+	// selfcheckTask = nil
 
 	return nil
 }
 
-func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
+func selfcheckTaskFunc(wc *mgr.WorkerCtx) error {
 	// Create tracing logger.
-	ctx, tracer := log.AddTracer(ctx)
+	ctx, tracer := log.AddTracer(wc.Ctx())
 	defer tracer.Submit()
 	tracer.Tracef("compat: running self-check")
 
@@ -115,7 +132,7 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 		}
 
 		// Retry quicker when failed.
-		task.Schedule(time.Now().Add(selfcheckTaskRetryAfter))
+		module.selfcheckWorkerMgr.Delay(selfcheckTaskRetryAfter)
 
 		return nil
 	}
@@ -135,3 +152,33 @@ func selfcheckTaskFunc(ctx context.Context, task *modules.Task) error {
 func SelfCheckIsFailing() bool {
 	return selfCheckIsFailing.IsSet()
 }
+
+var (
+	module     *Compat
+	shimLoaded atomic.Bool
+)
+
+// New returns a new Compat module.
+func New(instance instance) (*Compat, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Compat")
+	module = &Compat{
+		mgr:      m,
+		instance: instance,
+
+		selfcheckWorkerMgr:            m.NewWorkerMgr("compatibility self-check", selfcheckTaskFunc, nil),
+		cleanNotifyThresholdWorkerMgr: m.NewWorkerMgr("clean notify thresholds", cleanNotifyThreshold, nil),
+
+		states: mgr.NewStateMgr(m),
+	}
+	if err := prep(); err != nil {
+		return nil, err
+	}
+	return module, nil
+}
+
+type instance interface {
+	NetEnv() *netenv.NetEnv
+}

service/compat/notify.go

@@ -1,17 +1,16 @@
 package compat
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portbase/notifications"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/process"
 	"github.com/safing/portmaster/service/profile"
 )
@@ -112,7 +111,7 @@ func systemCompatOrManualDNSIssue() *systemIssue {
 	return manualDNSSetupRequired
 }
 
-func (issue *systemIssue) notify(err error) {
+func (issue *systemIssue) notify(err error) { //nolint // TODO: Should we use the error?
 	systemIssueNotificationLock.Lock()
 	defer systemIssueNotificationLock.Unlock()
 
@@ -138,10 +137,7 @@ func (issue *systemIssue) notify(err error) {
 	notifications.Notify(n)
 
 	systemIssueNotification = n
-	n.AttachToModule(module)
-
-	// Report the raw error as module error.
-	module.NewErrorMessage("selfcheck", err).Report()
+	n.SyncWithState(module.states)
 }
 
 func resetSystemIssue() {
@@ -214,7 +210,7 @@ func (issue *appIssue) notify(proc *process.Process) {
 	notifications.Notify(n)
 
 	// Set warning on profile.
-	module.StartWorker("set app compat warning", func(ctx context.Context) error {
+	module.mgr.Go("set app compat warning", func(ctx *mgr.WorkerCtx) error {
 		var changed bool
 
 		func() {
@@ -273,7 +269,7 @@ func isOverThreshold(id string) bool {
 	return false
 }
 
-func cleanNotifyThreshold(ctx context.Context, task *modules.Task) error {
+func cleanNotifyThreshold(ctx *mgr.WorkerCtx) error {
 	notifyThresholdsLock.Lock()
 	defer notifyThresholdsLock.Unlock()
 

service/compat/selfcheck.go

@@ -10,8 +10,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/rng"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/netenv"
 	"github.com/safing/portmaster/service/network/packet"
 	"github.com/safing/portmaster/service/resolver"
@@ -130,7 +131,7 @@ func selfcheck(ctx context.Context) (issue *systemIssue, err error) {
 	}
 
 	// Start worker for the DNS lookup.
-	module.StartWorker("dns check lookup", func(_ context.Context) error {
+	module.mgr.Go("dns check lookup", func(_ *mgr.WorkerCtx) error {
 		ips, err := net.LookupIP(randomSubdomain + DNSCheckInternalDomainScope)
 		if err == nil && len(ips) > 0 {
 			dnsCheckReturnedIP = ips[0]

service/compat/wfpstate.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"text/tabwriter"
 
-	"github.com/safing/portbase/utils/osdetail"
+	"github.com/safing/portmaster/base/utils/osdetail"
 )
 
 // GetWFPState queries the system for the WFP state and returns a simplified