Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

service/intel/geoip/database.go

@@ -1,15 +1,15 @@
 package geoip
 
 import (
-	"context"
 	"fmt"
 	"sync"
 	"time"
 
 	maxminddb "github.com/oschwald/maxminddb-golang"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/updater"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/updater"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/updates"
 )
 
@@ -148,11 +148,11 @@ func (upd *updateWorker) triggerUpdate() {
 
 func (upd *updateWorker) start() {
 	upd.once.Do(func() {
-		module.StartServiceWorker("geoip-updater", time.Second*10, upd.run)
+		module.mgr.Go("geoip-updater", upd.run)
 	})
 }
 
-func (upd *updateWorker) run(ctx context.Context) error {
+func (upd *updateWorker) run(ctx *mgr.WorkerCtx) error {
 	for {
 		if upd.v4.NeedsUpdate() {
 			if v4, err := getGeoIPDB(v4MMDBResource); err == nil {

service/intel/geoip/init_test.go

@@ -0,0 +1,109 @@
+package geoip
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/database/dbmodule"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/service/updates"
+)
+
+type testInstance struct {
+	db      *dbmodule.DBModule
+	api     *api.API
+	config  *config.Config
+	updates *updates.Updates
+}
+
+var _ instance = &testInstance{}
+
+func (stub *testInstance) Updates() *updates.Updates {
+	return stub.updates
+}
+
+func (stub *testInstance) API() *api.API {
+	return stub.api
+}
+
+func (stub *testInstance) Config() *config.Config {
+	return stub.config
+}
+
+func (stub *testInstance) Notifications() *notifications.Notifications {
+	return nil
+}
+
+func (stub *testInstance) Ready() bool {
+	return true
+}
+
+func (stub *testInstance) Restart() {}
+
+func (stub *testInstance) Shutdown() {}
+
+func (stub *testInstance) SetCmdLineOperation(f func() error) {}
+
+func runTest(m *testing.M) error {
+	api.SetDefaultAPIListenAddress("0.0.0.0:8080")
+	ds, err := config.InitializeUnitTestDataroot("test-geoip")
+	if err != nil {
+		return fmt.Errorf("failed to initialize dataroot: %w", err)
+	}
+	defer func() { _ = os.RemoveAll(ds) }()
+
+	stub := &testInstance{}
+	stub.db, err = dbmodule.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create database: %w", err)
+	}
+	stub.config, err = config.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create config: %w", err)
+	}
+	stub.api, err = api.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create api: %w", err)
+	}
+	stub.updates, err = updates.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create updates: %w", err)
+	}
+	module, err = New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to initialize module: %w", err)
+	}
+
+	err = stub.db.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start database: %w", err)
+	}
+	err = stub.config.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start config: %w", err)
+	}
+	err = stub.api.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start api: %w", err)
+	}
+	err = stub.updates.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start updates: %w", err)
+	}
+	err = module.Start()
+	if err != nil {
+		return fmt.Errorf("failed to start module: %w", err)
+	}
+	m.Run()
+	return nil
+}
+
+func TestMain(m *testing.M) {
+	if err := runTest(m); err != nil {
+		fmt.Printf("%s\n", err)
+		os.Exit(1)
+	}
+}

service/intel/geoip/location.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/umahmood/haversine"
 
-	"github.com/safing/portbase/utils"
+	"github.com/safing/portmaster/base/utils"
 )
 
 const (

service/intel/geoip/lookup_test.go

@@ -3,6 +3,7 @@ package geoip
 import (
 	"net"
 	"testing"
+	"time"
 )
 
 func TestLocationLookup(t *testing.T) {
@@ -12,6 +13,17 @@ func TestLocationLookup(t *testing.T) {
 	}
 	t.Parallel()
 
+	// Wait for db to be initialized
+	worker.v4.rw.Lock()
+	waiter := worker.v4.getWaiter()
+	worker.v4.rw.Unlock()
+
+	worker.triggerUpdate()
+	select {
+	case <-waiter:
+	case <-time.After(15 * time.Second):
+	}
+
 	ip1 := net.ParseIP("81.2.69.142")
 	loc1, err := GetLocation(ip1)
 	if err != nil {

service/intel/geoip/module.go

@@ -1,20 +1,54 @@
 package geoip
 
 import (
-	"context"
+	"errors"
+	"sync/atomic"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/updates"
 )
 
-var module *modules.Module
-
-func init() {
-	module = modules.Register("geoip", prep, nil, nil, "base", "updates")
+type GeoIP struct {
+	mgr      *mgr.Manager
+	instance instance
 }
 
-func prep() error {
+func (g *GeoIP) Manager() *mgr.Manager {
+	return g.mgr
+}
+
+func (g *GeoIP) Start() error {
+	module.instance.Updates().EventResourcesUpdated.AddCallback(
+		"Check for GeoIP database updates",
+		func(_ *mgr.WorkerCtx, _ struct{}) (bool, error) {
+			worker.triggerUpdate()
+			return false, nil
+		})
+	return nil
+}
+
+func (g *GeoIP) Stop() error {
+	return nil
+}
+
+var (
+	module     *GeoIP
+	shimLoaded atomic.Bool
+)
+
+// New returns a new GeoIP module.
+func New(instance instance) (*GeoIP, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+
+	m := mgr.New("geoip")
+	module = &GeoIP{
+		mgr:      m,
+		instance: instance,
+	}
+
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path: "intel/geoip/countries",
 		Read: api.PermitUser,
@@ -25,16 +59,12 @@ func prep() error {
 		Name:        "Get Country Information",
 		Description: "Returns a map of country information centers indexed by ISO-A2 country code",
 	}); err != nil {
-		return err
+		return nil, err
 	}
 
-	return module.RegisterEventHook(
-		updates.ModuleName,
-		updates.ResourceUpdateEvent,
-		"Check for GeoIP database updates",
-		func(c context.Context, i interface{}) error {
-			worker.triggerUpdate()
-			return nil
-		},
-	)
+	return module, nil
+}
+
+type instance interface {
+	Updates() *updates.Updates
 }

service/intel/geoip/module_test.go

@@ -1,11 +0,0 @@
-package geoip
-
-import (
-	"testing"
-
-	"github.com/safing/portmaster/service/core/pmtesting"
-)
-
-func TestMain(m *testing.M) {
-	pmtesting.TestMain(m, module)
-}

service/intel/geoip/regions.go

@@ -1,7 +1,7 @@
 package geoip
 
 import (
-	"github.com/safing/portbase/utils"
+	"github.com/safing/portmaster/base/utils"
 )
 
 // IsRegionalNeighbor returns whether the supplied location is a regional neighbor.

service/intel/geoip/regions_test.go

@@ -3,7 +3,7 @@ package geoip
 import (
 	"testing"
 
-	"github.com/safing/portbase/utils"
+	"github.com/safing/portmaster/base/utils"
 )
 
 func TestRegions(t *testing.T) {