Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

service/core/api.go

@@ -9,13 +9,12 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portbase/notifications"
-	"github.com/safing/portbase/rng"
-	"github.com/safing/portbase/utils/debug"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/base/rng"
+	"github.com/safing/portmaster/base/utils/debug"
 	"github.com/safing/portmaster/service/compat"
 	"github.com/safing/portmaster/service/process"
 	"github.com/safing/portmaster/service/resolver"
@@ -54,7 +53,6 @@ func registerAPIEndpoints() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:        "debug/core",
 		Read:        api.PermitAnyone,
-		BelongsTo:   module,
 		DataFunc:    debugInfo,
 		Name:        "Get Debug Information",
 		Description: "Returns network debugging information, similar to debug/info, but with system status data.",
@@ -71,7 +69,6 @@ func registerAPIEndpoints() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:       "app/auth",
 		Read:       api.PermitAnyone,
-		BelongsTo:  module,
 		StructFunc: authorizeApp,
 		Name:       "Request an authentication token with a given set of permissions. The user will be prompted to either authorize or deny the request. Used for external or third-party tool integrations.",
 		Parameters: []api.Parameter{
@@ -103,7 +100,6 @@ func registerAPIEndpoints() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:       "app/profile",
 		Read:       api.PermitUser,
-		BelongsTo:  module,
 		StructFunc: getMyProfile,
 		Name:       "Get the ID of the calling profile",
 	}); err != nil {
@@ -117,9 +113,7 @@ func registerAPIEndpoints() error {
 func shutdown(_ *api.Request) (msg string, err error) {
 	log.Warning("core: user requested shutdown via action")
 
-	// Do not run in worker, as this would block itself here.
-	go modules.Shutdown() //nolint:errcheck
-
+	module.instance.Shutdown()
 	return "shutdown initiated", nil
 }
 
@@ -145,8 +139,7 @@ func debugInfo(ar *api.Request) (data []byte, err error) {
 	di.AddVersionInfo()
 	di.AddPlatformInfo(ar.Context())
 
-	// Errors and unexpected logs.
-	di.AddLastReportedModuleError()
+	// Unexpected logs.
 	di.AddLastUnexpectedLogs()
 
 	// Status Information from various modules.

service/core/base/databases.go

@@ -1,9 +1,9 @@
 package base
 
 import (
-	"github.com/safing/portbase/database"
-	_ "github.com/safing/portbase/database/dbmodule"
-	_ "github.com/safing/portbase/database/storage/bbolt"
+	"github.com/safing/portmaster/base/database"
+	_ "github.com/safing/portmaster/base/database/dbmodule"
+	_ "github.com/safing/portmaster/base/database/storage/bbolt"
 )
 
 // Default Values (changeable for testing).

service/core/base/global.go

@@ -5,10 +5,10 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/dataroot"
-	"github.com/safing/portbase/info"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/dataroot"
+	"github.com/safing/portmaster/base/info"
+	"github.com/safing/portmaster/service/mgr"
 )
 
 // Default Values (changeable for testing).
@@ -24,11 +24,9 @@ func init() {
 	flag.StringVar(&dataDir, "data", "", "set data directory")
 	flag.StringVar(&databaseDir, "db", "", "alias to --data (deprecated)")
 	flag.BoolVar(&showVersion, "version", false, "show version and exit")
-
-	modules.SetGlobalPrepFn(globalPrep)
 }
 
-func globalPrep() error {
+func prep(instance instance) error {
 	// check if meta info is ok
 	err := info.CheckVersion()
 	if err != nil {
@@ -37,8 +35,8 @@ func globalPrep() error {
 
 	// print version
 	if showVersion {
-		fmt.Println(info.FullVersion())
-		return modules.ErrCleanExit
+		instance.SetCmdLineOperation(printVersion)
+		return mgr.ErrExecuteCmdLineOp
 	}
 
 	// check data root
@@ -67,3 +65,8 @@ func globalPrep() error {
 
 	return nil
 }
+
+func printVersion() error {
+	fmt.Println(info.FullVersion())
+	return nil
+}

service/core/base/logs.go

@@ -1,16 +1,15 @@
 package base
 
 import (
-	"context"
 	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/safing/portbase/dataroot"
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/dataroot"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 )
 
 const (
@@ -20,12 +19,10 @@ const (
 )
 
 func registerLogCleaner() {
-	module.NewTask("log cleaner", logCleaner).
-		Repeat(24 * time.Hour).
-		Schedule(time.Now().Add(15 * time.Minute))
+	_ = module.mgr.Delay("log cleaner", 15*time.Minute, logCleaner).Repeat(24 * time.Hour)
 }
 
-func logCleaner(_ context.Context, _ *modules.Task) error {
+func logCleaner(_ *mgr.WorkerCtx) error {
 	ageThreshold := time.Now().Add(-logTTL)
 
 	return filepath.Walk(

service/core/base/module.go

@@ -1,38 +1,62 @@
 package base
 
 import (
-	_ "github.com/safing/portbase/config"
-	_ "github.com/safing/portbase/metrics"
-	"github.com/safing/portbase/modules"
-	_ "github.com/safing/portbase/rng"
+	"errors"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/service/mgr"
 )
 
-var module *modules.Module
-
-func init() {
-	module = modules.Register("base", nil, start, nil, "database", "config", "rng", "metrics")
-
-	// For prettier subsystem graph, printed with --print-subsystem-graph
-	/*
-		subsystems.Register(
-			"base",
-			"Base",
-			"THE GROUND.",
-			baseModule,
-			"",
-			nil,
-		)
-	*/
+// Base is the base module.
+type Base struct {
+	mgr      *mgr.Manager
+	instance instance
 }
 
-func start() error {
+// Manager returns the module manager.
+func (b *Base) Manager() *mgr.Manager {
+	return b.mgr
+}
+
+// Start starts the module.
+func (b *Base) Start() error {
 	startProfiling()
-
-	if err := registerDatabases(); err != nil {
-		return err
-	}
-
 	registerLogCleaner()
 
 	return nil
 }
+
+// Stop stops the module.
+func (b *Base) Stop() error {
+	return nil
+}
+
+var (
+	module     *Base
+	shimLoaded atomic.Bool
+)
+
+// New returns a new Base module.
+func New(instance instance) (*Base, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Base")
+	module = &Base{
+		mgr:      m,
+		instance: instance,
+	}
+
+	if err := prep(instance); err != nil {
+		return nil, err
+	}
+	if err := registerDatabases(); err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
+
+type instance interface {
+	SetCmdLineOperation(f func() error)
+}

service/core/base/profiling.go

@@ -1,11 +1,12 @@
 package base
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
 	"runtime/pprof"
+
+	"github.com/safing/portmaster/service/mgr"
 )
 
 var cpuProfile string
@@ -16,11 +17,11 @@ func init() {
 
 func startProfiling() {
 	if cpuProfile != "" {
-		module.StartWorker("cpu profiler", cpuProfiler)
+		module.mgr.Go("cpu profiler", cpuProfiler)
 	}
 }
 
-func cpuProfiler(ctx context.Context) error {
+func cpuProfiler(ctx *mgr.WorkerCtx) error {
 	f, err := os.Create(cpuProfile)
 	if err != nil {
 		return fmt.Errorf("could not create CPU profile: %w", err)

service/core/config.go

@@ -6,8 +6,8 @@ import (
 	locale "github.com/Xuanwo/go-locale"
 	"golang.org/x/exp/slices"
 
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
 )
 
 // Configuration Keys.

service/core/core.go

@@ -1,58 +1,58 @@
 package core
 
 import (
+	"errors"
 	"flag"
 	"fmt"
-	"time"
+	"sync/atomic"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/metrics"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portbase/modules/subsystems"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/metrics"
 	_ "github.com/safing/portmaster/service/broadcasts"
+	"github.com/safing/portmaster/service/mgr"
 	_ "github.com/safing/portmaster/service/netenv"
 	_ "github.com/safing/portmaster/service/netquery"
 	_ "github.com/safing/portmaster/service/status"
 	_ "github.com/safing/portmaster/service/sync"
 	_ "github.com/safing/portmaster/service/ui"
-	"github.com/safing/portmaster/service/updates"
 )
 
-const (
-	eventShutdown = "shutdown"
-	eventRestart  = "restart"
-)
+// Core is the core service module.
+type Core struct {
+	m        *mgr.Manager
+	instance instance
 
-var (
-	module *modules.Module
+	EventShutdown *mgr.EventMgr[struct{}]
+	EventRestart  *mgr.EventMgr[struct{}]
+}
 
-	disableShutdownEvent bool
-)
+// Manager returns the manager.
+func (c *Core) Manager() *mgr.Manager {
+	return c.m
+}
+
+// Start starts the module.
+func (c *Core) Start() error {
+	return start()
+}
+
+// Stop stops the module.
+func (c *Core) Stop() error {
+	return nil
+}
+
+var disableShutdownEvent bool
 
 func init() {
-	module = modules.Register("core", prep, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui", "netenv", "network", "netquery", "interception", "compat", "broadcasts", "sync")
-	subsystems.Register(
-		"core",
-		"Core",
-		"Base Structure and System Integration",
-		module,
-		"config:core/",
-		nil,
-	)
-
 	flag.BoolVar(
 		&disableShutdownEvent,
 		"disable-shutdown-event",
 		false,
 		"disable shutdown event to keep app and notifier open when core shuts down",
 	)
-
-	modules.SetGlobalShutdownFn(shutdownHook)
 }
 
 func prep() error {
-	registerEvents()
-
 	// init config
 	err := registerConfig()
 	if err != nil {
@@ -63,6 +63,10 @@ func prep() error {
 		return err
 	}
 
+	if err := initModulesIntegration(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -79,22 +83,33 @@ func start() error {
 	return nil
 }
 
-func registerEvents() {
-	module.RegisterEvent(eventShutdown, true)
-	module.RegisterEvent(eventRestart, true)
-}
+var (
+	module     *Core
+	shimLoaded atomic.Bool
+)
 
-func shutdownHook() {
-	// Notify everyone of the restart/shutdown.
-	if !updates.IsRestarting() {
-		// Only trigger shutdown event if not disabled.
-		if !disableShutdownEvent {
-			module.TriggerEvent(eventShutdown, nil)
-		}
-	} else {
-		module.TriggerEvent(eventRestart, nil)
+// New returns a new NetEnv module.
+func New(instance instance) (*Core, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
 	}
 
-	// Wait a bit for the event to propagate.
-	time.Sleep(100 * time.Millisecond)
+	m := mgr.New("Core")
+	module = &Core{
+		m:        m,
+		instance: instance,
+
+		EventShutdown: mgr.NewEventMgr[struct{}]("shutdown", m),
+		EventRestart:  mgr.NewEventMgr[struct{}]("restart", m),
+	}
+
+	if err := prep(); err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
+
+type instance interface {
+	Shutdown()
 }

service/core/events.go

@@ -0,0 +1,79 @@
+package core
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/runtime"
+	"github.com/safing/portmaster/service/mgr"
+)
+
+var modulesIntegrationUpdatePusher func(...record.Record)
+
+func initModulesIntegration() (err error) {
+	modulesIntegrationUpdatePusher, err = runtime.Register("modules/", &ModulesIntegration{})
+	if err != nil {
+		return err
+	}
+
+	// Push events via API.
+	module.EventRestart.AddCallback("expose restart event", func(wc *mgr.WorkerCtx, s struct{}) (cancel bool, err error) {
+		// Send event as runtime:modules/core/event/restart
+		pushModuleEvent("core", "restart", false, nil)
+		return false, nil
+	})
+	module.EventShutdown.AddCallback("expose shutdown event", func(wc *mgr.WorkerCtx, s struct{}) (cancel bool, err error) {
+		// Send event as runtime:modules/core/event/shutdown
+		pushModuleEvent("core", "shutdown", false, nil)
+		return false, nil
+	})
+
+	return nil
+}
+
+// ModulesIntegration provides integration with the modules system.
+type ModulesIntegration struct{}
+
+// Set is called when the value is set from outside.
+// If the runtime value is considered read-only ErrReadOnly
+// should be returned. It is guaranteed that the key of
+// the record passed to Set is prefixed with the key used
+// to register the value provider.
+func (mi *ModulesIntegration) Set(record.Record) (record.Record, error) {
+	return nil, runtime.ErrReadOnly
+}
+
+// Get should return one or more records that match keyOrPrefix.
+// keyOrPrefix is guaranteed to be at least the prefix used to
+// register the ValueProvider.
+func (mi *ModulesIntegration) Get(keyOrPrefix string) ([]record.Record, error) {
+	return nil, database.ErrNotFound
+}
+
+type eventData struct {
+	record.Base
+	sync.Mutex
+	Data interface{}
+}
+
+func pushModuleEvent(moduleName, eventName string, internal bool, data interface{}) {
+	// Create event record and set key.
+	eventRecord := &eventData{
+		Data: data,
+	}
+	eventRecord.SetKey(fmt.Sprintf(
+		"runtime:modules/%s/event/%s",
+		moduleName,
+		eventName,
+	))
+	eventRecord.UpdateMeta()
+	if internal {
+		eventRecord.Meta().MakeSecret()
+		eventRecord.Meta().MakeCrownJewel()
+	}
+
+	// Push event to database subscriptions.
+	modulesIntegrationUpdatePusher(eventRecord)
+}

service/core/os_windows.go

@@ -1,8 +1,8 @@
 package core
 
 import (
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils/osdetail"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/utils/osdetail"
 )
 
 // only return on Fatal error!

service/core/pmtesting/testing.go

@@ -1,137 +0,0 @@
-// Package pmtesting provides a simple unit test setup routine.
-//
-// Usage:
-//
-//	package name
-//
-//	import (
-//		"testing"
-//
-//		"github.com/safing/portmaster/service/core/pmtesting"
-//	)
-//
-//	func TestMain(m *testing.M) {
-//		pmtesting.TestMain(m, module)
-//	}
-package pmtesting
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime/pprof"
-	"testing"
-
-	_ "github.com/safing/portbase/database/storage/hashmap"
-	"github.com/safing/portbase/dataroot"
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portmaster/service/core/base"
-)
-
-var printStackOnExit bool
-
-func init() {
-	flag.BoolVar(&printStackOnExit, "print-stack-on-exit", false, "prints the stack before of shutting down")
-}
-
-// TestHookFunc describes the functions passed to TestMainWithHooks.
-type TestHookFunc func() error
-
-// TestMain provides a simple unit test setup routine.
-func TestMain(m *testing.M, module *modules.Module) {
-	TestMainWithHooks(m, module, nil, nil)
-}
-
-// TestMainWithHooks provides a simple unit test setup routine and calls
-// afterStartFn after modules have started and beforeStopFn before modules
-// are shutdown.
-func TestMainWithHooks(m *testing.M, module *modules.Module, afterStartFn, beforeStopFn TestHookFunc) {
-	// Only enable needed modules.
-	modules.EnableModuleManagement(nil)
-
-	// Enable this module for testing.
-	if module != nil {
-		module.Enable()
-	}
-
-	// switch databases to memory only
-	base.DefaultDatabaseStorageType = "hashmap"
-
-	// switch API to high port
-	base.DefaultAPIListenAddress = "127.0.0.1:10817"
-
-	// set log level
-	log.SetLogLevel(log.TraceLevel)
-
-	// tmp dir for data root (db & config)
-	tmpDir := filepath.Join(os.TempDir(), "portmaster-testing")
-	// initialize data dir
-	err := dataroot.Initialize(tmpDir, 0o0755)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to initialize data root: %s\n", err)
-		os.Exit(1)
-	}
-
-	// start modules
-	var exitCode int
-	err = modules.Start()
-	if err != nil {
-		// starting failed
-		fmt.Fprintf(os.Stderr, "failed to setup test: %s\n", err)
-		exitCode = 1
-	} else {
-		runTests := true
-		if afterStartFn != nil {
-			if err := afterStartFn(); err != nil {
-				fmt.Fprintf(os.Stderr, "failed to run test start hook: %s\n", err)
-				runTests = false
-				exitCode = 1
-			}
-		}
-
-		if runTests {
-			// run tests
-			exitCode = m.Run()
-		}
-	}
-
-	if beforeStopFn != nil {
-		if err := beforeStopFn(); err != nil {
-			fmt.Fprintf(os.Stderr, "failed to run test shutdown hook: %s\n", err)
-		}
-	}
-
-	// shutdown
-	_ = modules.Shutdown()
-	if modules.GetExitStatusCode() != 0 {
-		exitCode = modules.GetExitStatusCode()
-		fmt.Fprintf(os.Stderr, "failed to cleanly shutdown test: %s\n", err)
-	}
-	printStack()
-
-	// clean up and exit
-
-	// Important: Do not remove tmpDir, as it is used as a cache for updates.
-	// remove config
-	_ = os.Remove(filepath.Join(tmpDir, "config.json"))
-	// remove databases
-	_ = os.Remove(filepath.Join(tmpDir, "databases.json"))
-	_ = os.RemoveAll(filepath.Join(tmpDir, "databases"))
-
-	os.Exit(exitCode)
-}
-
-func printStack() {
-	if printStackOnExit {
-		fmt.Println("=== PRINTING TRACES ===")
-		fmt.Println("=== GOROUTINES ===")
-		_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 2)
-		fmt.Println("=== BLOCKING ===")
-		_ = pprof.Lookup("block").WriteTo(os.Stdout, 2)
-		fmt.Println("=== MUTEXES ===")
-		_ = pprof.Lookup("mutex").WriteTo(os.Stdout, 2)
-		fmt.Println("=== END TRACES ===")
-	}
-}