Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

spn/hub/database.go

@@ -6,10 +6,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/database/iterator"
-	"github.com/safing/portbase/database/query"
-	"github.com/safing/portbase/database/record"
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/iterator"
+	"github.com/safing/portmaster/base/database/query"
+	"github.com/safing/portmaster/base/database/record"
 )
 
 var (

spn/hub/hub.go

@@ -9,8 +9,8 @@ import (
 	"golang.org/x/exp/slices"
 
 	"github.com/safing/jess"
-	"github.com/safing/portbase/database/record"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/profile/endpoints"
 )
 
@@ -425,7 +425,7 @@ func equalStringSlice(a, b []string) bool {
 		return false
 	}
 
-	for i := 0; i < len(a); i++ {
+	for i := range len(a) {
 		if a[i] != b[i] {
 			return false
 		}

spn/hub/hub_test.go

@@ -1,20 +1,121 @@
 package hub
 
 import (
+	"fmt"
 	"net"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/safing/portbase/modules"
-	_ "github.com/safing/portmaster/service/core/base"
-	"github.com/safing/portmaster/service/core/pmtesting"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/database/dbmodule"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/service/core/base"
+	"github.com/safing/portmaster/service/updates"
 )
 
+type testInstance struct {
+	db      *dbmodule.DBModule
+	api     *api.API
+	config  *config.Config
+	updates *updates.Updates
+	base    *base.Base
+}
+
+func (stub *testInstance) Updates() *updates.Updates {
+	return stub.updates
+}
+
+func (stub *testInstance) API() *api.API {
+	return stub.api
+}
+
+func (stub *testInstance) Config() *config.Config {
+	return stub.config
+}
+
+func (stub *testInstance) Notifications() *notifications.Notifications {
+	return nil
+}
+
+func (stub *testInstance) Base() *base.Base {
+	return stub.base
+}
+
+func (stub *testInstance) Ready() bool {
+	return true
+}
+
+func (stub *testInstance) Restart() {}
+
+func (stub *testInstance) Shutdown() {}
+
+func (stub *testInstance) SetCmdLineOperation(f func() error) {}
+
+func runTest(m *testing.M) error {
+	api.SetDefaultAPIListenAddress("0.0.0.0:8080")
+	ds, err := config.InitializeUnitTestDataroot("test-hub")
+	if err != nil {
+		return fmt.Errorf("failed to initialize dataroot: %w", err)
+	}
+	defer func() { _ = os.RemoveAll(ds) }()
+
+	stub := &testInstance{}
+	// Init
+	stub.db, err = dbmodule.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create database: %w", err)
+	}
+	stub.api, err = api.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create api: %w", err)
+	}
+	stub.config, err = config.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create config: %w", err)
+	}
+	stub.updates, err = updates.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create updates: %w", err)
+	}
+	stub.base, err = base.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to base updates: %w", err)
+	}
+
+	// Start
+	err = stub.db.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start database: %w", err)
+	}
+	err = stub.api.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start api: %w", err)
+	}
+	err = stub.config.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start config: %w", err)
+	}
+	err = stub.updates.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start updates: %w", err)
+	}
+	err = stub.base.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start base: %w", err)
+	}
+
+	m.Run()
+	return nil
+}
+
 func TestMain(m *testing.M) {
-	// TODO: We need the database module, so maybe set up a module for this package.
-	module := modules.Register("hub", nil, nil, nil, "base")
-	pmtesting.TestMain(m, module)
+	if err := runTest(m); err != nil {
+		fmt.Printf("%s", err)
+		os.Exit(1)
+	}
 }
 
 func TestEquality(t *testing.T) {

spn/hub/update.go

@@ -7,11 +7,11 @@ import (
 
 	"github.com/safing/jess"
 	"github.com/safing/jess/lhash"
-	"github.com/safing/portbase/container"
-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/formats/dsd"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/network/netutils"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
 )
 
 var (

spn/hub/update_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/safing/jess"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/dsd"
 )
 
 func TestHubUpdate(t *testing.T) {