Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

service/netenv/adresses.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/network/netutils"
 )
 

service/netenv/api.go

@@ -3,14 +3,13 @@ package netenv
 import (
 	"errors"
 
-	"github.com/safing/portbase/api"
+	"github.com/safing/portmaster/base/api"
 )
 
 func registerAPIEndpoints() error {
 	if err := api.RegisterEndpoint(api.Endpoint{
-		Path:      "network/gateways",
-		Read:      api.PermitUser,
-		BelongsTo: module,
+		Path: "network/gateways",
+		Read: api.PermitUser,
 		StructFunc: func(ar *api.Request) (i interface{}, err error) {
 			return Gateways(), nil
 		},
@@ -21,9 +20,8 @@ func registerAPIEndpoints() error {
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
-		Path:      "network/nameservers",
-		Read:      api.PermitUser,
-		BelongsTo: module,
+		Path: "network/nameservers",
+		Read: api.PermitUser,
 		StructFunc: func(ar *api.Request) (i interface{}, err error) {
 			return Nameservers(), nil
 		},
@@ -34,9 +32,8 @@ func registerAPIEndpoints() error {
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
-		Path:      "network/location",
-		Read:      api.PermitUser,
-		BelongsTo: module,
+		Path: "network/location",
+		Read: api.PermitUser,
 		StructFunc: func(ar *api.Request) (i interface{}, err error) {
 			locs, ok := GetInternetLocation()
 			if ok {
@@ -51,9 +48,8 @@ func registerAPIEndpoints() error {
 	}
 
 	if err := api.RegisterEndpoint(api.Endpoint{
-		Path:      "network/location/traceroute",
-		Read:      api.PermitUser,
-		BelongsTo: module,
+		Path: "network/location/traceroute",
+		Read: api.PermitUser,
 		StructFunc: func(ar *api.Request) (i interface{}, err error) {
 			return getLocationFromTraceroute(&DeviceLocations{})
 		},

service/netenv/dbus_linux.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/godbus/dbus/v5"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 )
 
 var (

service/netenv/environment_linux.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/miekg/dns"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/network/netutils"
 )
 

service/netenv/environment_windows.go

@@ -10,8 +10,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils/osdetail"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/utils/osdetail"
 )
 
 // Gateways returns the currently active gateways.

service/netenv/icmp_listener.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/tevino/abool"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/network/packet"
 )
 

service/netenv/init_test.go

@@ -0,0 +1,106 @@
+package netenv
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/database/dbmodule"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/service/updates"
+)
+
+type testInstance struct {
+	db      *dbmodule.DBModule
+	api     *api.API
+	config  *config.Config
+	updates *updates.Updates
+}
+
+var _ instance = &testInstance{}
+
+func (stub *testInstance) Updates() *updates.Updates {
+	return stub.updates
+}
+
+func (stub *testInstance) API() *api.API {
+	return stub.api
+}
+
+func (stub *testInstance) Config() *config.Config {
+	return stub.config
+}
+
+func (stub *testInstance) Notifications() *notifications.Notifications {
+	return nil
+}
+
+func (stub *testInstance) Ready() bool {
+	return true
+}
+
+func (stub *testInstance) Restart() {}
+
+func (stub *testInstance) Shutdown() {}
+
+func (stub *testInstance) SetCmdLineOperation(f func() error) {}
+
+func runTest(m *testing.M) error {
+	api.SetDefaultAPIListenAddress("0.0.0.0:8080")
+	ds, err := config.InitializeUnitTestDataroot("test-netenv")
+	if err != nil {
+		return fmt.Errorf("failed to initialize dataroot: %w", err)
+	}
+	defer func() { _ = os.RemoveAll(ds) }()
+
+	stub := &testInstance{}
+	stub.db, err = dbmodule.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create database: %w", err)
+	}
+	stub.config, err = config.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create config: %w", err)
+	}
+	stub.api, err = api.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create api: %w", err)
+	}
+	stub.updates, err = updates.New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to create updates: %w", err)
+	}
+	err = stub.db.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start database: %w", err)
+	}
+	err = stub.config.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start config: %w", err)
+	}
+	err = stub.api.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start api: %w", err)
+	}
+	err = stub.updates.Start()
+	if err != nil {
+		return fmt.Errorf("Failed to start updates: %w", err)
+	}
+
+	_, err = New(stub)
+	if err != nil {
+		return fmt.Errorf("failed to initialize module %w", err)
+	}
+
+	m.Run()
+	return nil
+}
+
+func TestMain(m *testing.M) {
+	if err := runTest(m); err != nil {
+		fmt.Printf("%s", err)
+		os.Exit(1)
+	}
+}

service/netenv/location.go

@@ -12,8 +12,8 @@ import (
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/rng"
 	"github.com/safing/portmaster/service/intel/geoip"
 	"github.com/safing/portmaster/service/network/netutils"
 	"github.com/safing/portmaster/service/network/packet"
@@ -418,7 +418,7 @@ nextHop:
 			// Send ICMP packet.
 			// Try to send three times, as this can be flaky.
 		sendICMP:
-			for i := 0; i < 3; i++ {
+			for range 3 {
 				_, err = conn.WriteTo(pingPacket, locationTestingIPv4Addr)
 				if err == nil {
 					break sendICMP

service/netenv/main.go

@@ -1,10 +1,13 @@
 package netenv
 
 import (
+	"errors"
+	"sync/atomic"
+
 	"github.com/tevino/abool"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 )
 
 // Event Names.
@@ -14,12 +17,34 @@ const (
 	OnlineStatusChangedEvent = "online status changed"
 )
 
-var module *modules.Module
+type NetEnv struct {
+	m        *mgr.Manager
+	instance instance
 
-func init() {
-	module = modules.Register(ModuleName, prep, start, nil)
-	module.RegisterEvent(NetworkChangedEvent, true)
-	module.RegisterEvent(OnlineStatusChangedEvent, true)
+	EventNetworkChange      *mgr.EventMgr[struct{}]
+	EventOnlineStatusChange *mgr.EventMgr[OnlineStatus]
+}
+
+func (ne *NetEnv) Manager() *mgr.Manager {
+	return ne.m
+}
+
+func (ne *NetEnv) Start() error {
+	ne.m.Go(
+		"monitor network changes",
+		monitorNetworkChanges,
+	)
+
+	ne.m.Go(
+		"monitor online status",
+		monitorOnlineStatus,
+	)
+
+	return nil
+}
+
+func (ne *NetEnv) Stop() error {
+	return nil
 }
 
 func prep() error {
@@ -36,22 +61,6 @@ func prep() error {
 	return prepLocation()
 }
 
-func start() error {
-	module.StartServiceWorker(
-		"monitor network changes",
-		0,
-		monitorNetworkChanges,
-	)
-
-	module.StartServiceWorker(
-		"monitor online status",
-		0,
-		monitorOnlineStatus,
-	)
-
-	return nil
-}
-
 var ipv6Enabled = abool.NewBool(true)
 
 // IPv6Enabled returns whether the device has an active IPv6 stack.
@@ -70,3 +79,30 @@ func checkForIPv6Stack() {
 	// Set IPv6 as enabled if any IPv6 addresses are found.
 	ipv6Enabled.SetTo(len(v6IPs) > 0)
 }
+
+var (
+	module     *NetEnv
+	shimLoaded atomic.Bool
+)
+
+// New returns a new NetEnv module.
+func New(instance instance) (*NetEnv, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("NetEnv")
+	module = &NetEnv{
+		m:        m,
+		instance: instance,
+
+		EventNetworkChange:      mgr.NewEventMgr[struct{}]("network change", m),
+		EventOnlineStatusChange: mgr.NewEventMgr[OnlineStatus]("online status change", m),
+	}
+	if err := prep(); err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
+
+type instance interface{}

service/netenv/main_test.go

@@ -1,11 +1,5 @@
 package netenv
 
-import (
-	"testing"
-
-	"github.com/safing/portmaster/service/core/pmtesting"
-)
-
-func TestMain(m *testing.M) {
-	pmtesting.TestMain(m, module)
-}
+// func TestMain(m *testing.M) {
+// 	pmtesting.TestMain(m, module)
+// }

service/netenv/network-change.go

@@ -2,13 +2,13 @@ package netenv
 
 import (
 	"bytes"
-	"context"
 	"crypto/sha1"
 	"io"
 	"time"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/utils"
+	"github.com/safing/portmaster/service/mgr"
 )
 
 var (
@@ -23,7 +23,7 @@ func GetNetworkChangedFlag() *utils.Flag {
 
 func notifyOfNetworkChange() {
 	networkChangedBroadcastFlag.NotifyAndReset()
-	module.TriggerEvent(NetworkChangedEvent, nil)
+	module.EventNetworkChange.Submit(struct{}{})
 }
 
 // TriggerNetworkChangeCheck triggers a network change check.
@@ -34,7 +34,7 @@ func TriggerNetworkChangeCheck() {
 	}
 }
 
-func monitorNetworkChanges(ctx context.Context) error {
+func monitorNetworkChanges(ctx *mgr.WorkerCtx) error {
 	var lastNetworkChecksum []byte
 
 serviceLoop:

service/netenv/online-status.go

@@ -13,8 +13,9 @@ import (
 
 	"github.com/tevino/abool"
 
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/notifications"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/notifications"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/network/netutils"
 	"github.com/safing/portmaster/service/updates"
 )
@@ -207,7 +208,7 @@ func updateOnlineStatus(status OnlineStatus, portalURL *url.URL, comment string)
 
 	// Trigger events.
 	if changed {
-		module.TriggerEvent(OnlineStatusChangedEvent, status)
+		module.EventOnlineStatusChange.Submit(status)
 		if status == StatusPortal {
 			log.Infof(`netenv: setting online status to %s at "%s" (%s)`, status, portalURL, comment)
 		} else {
@@ -356,7 +357,7 @@ func TriggerOnlineStatusInvestigation() {
 	}
 }
 
-func monitorOnlineStatus(ctx context.Context) error {
+func monitorOnlineStatus(ctx *mgr.WorkerCtx) error {
 	TriggerOnlineStatusInvestigation()
 	for {
 		// wait for trigger
@@ -372,7 +373,7 @@ func monitorOnlineStatus(ctx context.Context) error {
 			onlineStatusInvestigationWg.Add(1)
 		}
 
-		checkOnlineStatus(ctx)
+		checkOnlineStatus(ctx.Ctx())
 
 		// finished!
 		onlineStatusInvestigationWg.Done()