Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

spn/access/token/module_test.go

@@ -1,13 +1,26 @@
 package token
 
 import (
+	"fmt"
+	"os"
 	"testing"
 
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portmaster/service/core/pmtesting"
+	"github.com/safing/portmaster/base/rng"
 )
 
+type testInstance struct{}
+
 func TestMain(m *testing.M) {
-	module := modules.Register("token", nil, nil, nil, "rng")
-	pmtesting.TestMain(m, module)
+	rng, err := rng.New(testInstance{})
+	if err != nil {
+		fmt.Printf("failed to create RNG module: %s", err)
+		os.Exit(1)
+	}
+
+	err = rng.Start()
+	if err != nil {
+		fmt.Printf("failed to start RNG module: %s", err)
+		os.Exit(1)
+	}
+	m.Run()
 }

spn/access/token/pblind.go

@@ -13,8 +13,8 @@ import (
 	"github.com/mr-tron/base58"
 	"github.com/rot256/pblind"
 
-	"github.com/safing/portbase/container"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
 )
 
 const pblindSecretSize = 32
@@ -217,7 +217,7 @@ func (pbh *PBlindHandler) CreateSetup() (state *PBlindSignerState, setupResponse
 	}
 
 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		info, err := pbh.makeInfo(i + 1)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create info #%d: %w", i, err)
@@ -257,7 +257,7 @@ func (pbh *PBlindHandler) CreateTokenRequest(requestSetup *PBlindSetupResponse)
 	}
 
 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Check if we have setup data.
 		if requestSetup.Msgs[i] == nil {
 			return nil, fmt.Errorf("missing setup data #%d", i)
@@ -319,7 +319,7 @@ func (pbh *PBlindHandler) IssueTokens(state *PBlindSignerState, request *PBlindT
 	}
 
 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Check if we have request data.
 		if request.Msgs[i] == nil {
 			return nil, fmt.Errorf("missing request data #%d", i)
@@ -360,7 +360,7 @@ func (pbh *PBlindHandler) ProcessIssuedTokens(issuedTokens *IssuedPBlindTokens)
 	finalizedTokens := make([]*PBlindToken, pbh.opts.BatchSize)
 
 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Finalize token.
 		err := pbh.requestState[i].State.ProcessMessage3(*issuedTokens.Msgs[i])
 		if err != nil {

spn/access/token/pblind_test.go

@@ -124,7 +124,7 @@ func TestPBlindLibrary(t *testing.T) {
 
 	// Create signers and prep requests.
 	start := time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		signer, err := pblind.CreateSigner(sk, info)
 		if err != nil {
 			t.Fatal(err)
@@ -146,7 +146,7 @@ func TestPBlindLibrary(t *testing.T) {
 
 	// Create requesters and create requests.
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		requester, err := pblind.CreateRequester(pk, info, msgStr)
 		if err != nil {
 			t.Fatal(err)
@@ -178,7 +178,7 @@ func TestPBlindLibrary(t *testing.T) {
 
 	// Sign requests
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var msg2S pblind.Message2
 		_, err = asn1.Unmarshal(toServer[i], &msg2S)
 		if err != nil {
@@ -204,7 +204,7 @@ func TestPBlindLibrary(t *testing.T) {
 
 	// Verify signed requests
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var msg3R pblind.Message3
 		_, err := asn1.Unmarshal(toClient[i], &msg3R)
 		if err != nil {
@@ -234,7 +234,7 @@ func TestPBlindLibrary(t *testing.T) {
 
 	// Verify on server
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var sig pblind.Signature
 		_, err := asn1.Unmarshal(toServer[i], &sig)
 		if err != nil {

spn/access/token/request_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/dsd"
 )
 
 func TestFull(t *testing.T) {

spn/access/token/scramble.go

@@ -7,7 +7,7 @@ import (
 	"github.com/mr-tron/base58"
 
 	"github.com/safing/jess/lhash"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/dsd"
 )
 
 const (

spn/access/token/token.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/mr-tron/base58"
 
-	"github.com/safing/portbase/container"
+	"github.com/safing/structures/container"
 )
 
 // Token represents a token, consisting of a zone (name) and some data.

spn/access/token/token_test.go

@@ -3,7 +3,7 @@ package token
 import (
 	"testing"
 
-	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/base/rng"
 )
 
 func TestToken(t *testing.T) {