Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

spn/ships/connection_test.go

@@ -35,8 +35,7 @@ func TestConnections(t *testing.T) {
 		registryLock.Unlock()
 	})
 
-	for k, v := range registry { //nolint:paralleltest // False positive.
-		protocol, builder := k, v
+	for protocol, builder := range registry {
 		t.Run(protocol, func(t *testing.T) {
 			t.Parallel()
 
@@ -85,7 +84,7 @@ func TestConnections(t *testing.T) {
 			assert.Equal(t, testData, buf, "should match")
 			fmt.Print(".")
 
-			for i := 0; i < 100; i++ {
+			for range 100 {
 				// server send
 				err = srvShip.Load(testData)
 				if err != nil {

spn/ships/http.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/conf"
 	"github.com/safing/portmaster/spn/hub"
 )

spn/ships/http_info.go

@@ -6,9 +6,9 @@ import (
 	"html/template"
 	"net/http"
 
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/info"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/info"
+	"github.com/safing/portmaster/base/log"
 )
 
 var (

spn/ships/http_info_test.go

@@ -4,7 +4,7 @@ import (
 	"html/template"
 	"testing"
 
-	"github.com/safing/portbase/config"
+	"github.com/safing/portmaster/base/config"
 )
 
 func TestInfoPageTemplate(t *testing.T) {

spn/ships/http_shared.go

@@ -9,7 +9,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/spn/conf"
 )
 
@@ -96,8 +97,7 @@ func addHTTPHandler(port uint16, path string, handler http.HandlerFunc) error {
 		WriteTimeout:      1 * time.Minute,
 		IdleTimeout:       1 * time.Minute,
 		MaxHeaderBytes:    4096,
-		// ErrorLog:          &log.Logger{}, // FIXME
-		BaseContext: func(net.Listener) context.Context { return module.Ctx },
+		BaseContext:       func(net.Listener) context.Context { return module.mgr.Ctx() },
 	}
 	shared.server = server
 
@@ -121,11 +121,10 @@ func addHTTPHandler(port uint16, path string, handler http.HandlerFunc) error {
 	sharedHTTPServers[port] = shared
 
 	// Start servers in service workers.
-	for _, listener := range listeners {
-		serviceListener := listener
-		module.StartServiceWorker(
-			fmt.Sprintf("shared http server listener on %s", listener.Addr()), 0,
-			func(ctx context.Context) error {
+	for _, serviceListener := range listeners {
+		module.mgr.Go(
+			fmt.Sprintf("shared http server listener on %s", serviceListener.Addr()),
+			func(_ *mgr.WorkerCtx) error {
 				err := shared.server.Serve(serviceListener)
 				if !errors.Is(http.ErrServerClosed, err) {
 					return err

spn/ships/http_shared_test.go

@@ -8,10 +8,15 @@ import (
 )
 
 func TestSharedHTTP(t *testing.T) { //nolint:paralleltest // Test checks global state.
+	_, err := New(struct{}{})
+	if err != nil {
+		t.Errorf("failed to create module ships: %s", err)
+	}
+
 	const testPort = 65100
 
 	// Register multiple handlers.
-	err := addHTTPHandler(testPort, "", ServeInfoPage)
+	err = addHTTPHandler(testPort, "", ServeInfoPage)
 	require.NoError(t, err, "should be able to share http listener")
 	err = addHTTPHandler(testPort, "/test", ServeInfoPage)
 	require.NoError(t, err, "should be able to share http listener")

spn/ships/launch.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/netenv"
 	"github.com/safing/portmaster/spn/hub"
 )

spn/ships/module.go

@@ -1,20 +1,50 @@
 package ships
 
 import (
-	"github.com/safing/portbase/modules"
+	"errors"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/spn/conf"
 )
 
-var module *modules.Module
-
-func init() {
-	module = modules.Register("ships", start, nil, nil, "cabin")
+type Ships struct {
+	mgr      *mgr.Manager
+	instance instance
 }
 
-func start() error {
+func (s *Ships) Manager() *mgr.Manager {
+	return s.mgr
+}
+
+func (s *Ships) Start() error {
 	if conf.PublicHub() {
 		initPageInput()
 	}
 
 	return nil
 }
+
+func (s *Ships) Stop() error {
+	return nil
+}
+
+var (
+	module     *Ships
+	shimLoaded atomic.Bool
+)
+
+// New returns a new Config module.
+func New(instance instance) (*Ships, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Ships")
+	module = &Ships{
+		mgr:      m,
+		instance: instance,
+	}
+	return module, nil
+}
+
+type instance interface{}

spn/ships/ship.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/tevino/abool"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/hub"
 )
 

spn/ships/tcp.go

@@ -6,7 +6,8 @@ import (
 	"net"
 	"time"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/spn/conf"
 	"github.com/safing/portmaster/spn/hub"
 )
@@ -81,7 +82,7 @@ func establishTCPPier(transport *hub.Transport, dockingRequests chan Ship) (Pier
 	}
 
 	// Create new pier.
-	pierCtx, cancelCtx := context.WithCancel(module.Ctx)
+	pierCtx, cancelCtx := context.WithCancel(module.mgr.Ctx())
 	pier := &TCPPier{
 		PierBase: PierBase{
 			transport:       transport,
@@ -95,9 +96,8 @@ func establishTCPPier(transport *hub.Transport, dockingRequests chan Ship) (Pier
 
 	// Start workers.
 	for _, listener := range pier.listeners {
-		serviceListener := listener
-		module.StartServiceWorker("accept TCP docking requests", 0, func(ctx context.Context) error {
-			return pier.dockingWorker(ctx, serviceListener)
+		module.mgr.Go("accept TCP docking requests", func(wc *mgr.WorkerCtx) error {
+			return pier.dockingWorker(wc.Ctx(), listener)
 		})
 	}
 

spn/ships/testship_test.go

@@ -17,7 +17,7 @@ func TestTestShip(t *testing.T) {
 
 	srvShip := tShip.Reverse()
 
-	for i := 0; i < 100; i++ {
+	for range 100 {
 		// client send
 		err := ship.Load(testData)
 		if err != nil {