Restructure modules (#1572)

* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
2024-08-09 17:15:48 +02:00
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
--- a/spn/access/api.go
+++ b/spn/access/api.go
@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"net/http"

-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/database/record"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/access/account"
 )

--- a/spn/access/client.go
+++ b/spn/access/client.go
@@ -8,11 +8,11 @@ import (
 	"sync"
 	"time"

-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/formats/dsd"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/access/account"
 	"github.com/safing/portmaster/spn/access/token"
+	"github.com/safing/structures/dsd"
 )

 // Client URLs.
@@ -57,15 +57,8 @@ func makeClientRequest(opts *clientRequestOptions) (resp *http.Response, err err
 	// Get context for request.
 	var ctx context.Context
 	var cancel context.CancelFunc
-	if module.Online() {
-		// Only use module context if online.
-		ctx, cancel = context.WithTimeout(module.Ctx, opts.requestTimeout)
-		defer cancel()
-	} else {
-		// Otherwise, use the background context.
-		ctx, cancel = context.WithTimeout(context.Background(), opts.requestTimeout)
-		defer cancel()
-	}
+	ctx, cancel = context.WithTimeout(module.mgr.Ctx(), opts.requestTimeout)
+	defer cancel()

 	// Create new request.
 	request, err := http.NewRequestWithContext(ctx, opts.method, opts.url, nil)
@@ -214,7 +207,7 @@ func Login(username, password string) (user *UserRecord, code int, err error) {
 	defer clientRequestLock.Unlock()

 	// Trigger account update when done.
-	defer module.TriggerEvent(AccountUpdateEvent, nil)
+	defer module.EventAccountUpdate.Submit(struct{}{})

 	// Get previous user.
 	previousUser, err := GetUser()
@@ -300,7 +293,7 @@ func Logout(shallow, purge bool) error {
 	defer clientRequestLock.Unlock()

 	// Trigger account update when done.
-	defer module.TriggerEvent(AccountUpdateEvent, nil)
+	defer module.EventAccountUpdate.Submit(struct{}{})

 	// Clear caches.
 	clearUserCaches()
@@ -382,7 +375,7 @@ func UpdateUser() (user *UserRecord, statusCode int, err error) {
 	defer clientRequestLock.Unlock()

 	// Trigger account update when done.
-	defer module.TriggerEvent(AccountUpdateEvent, nil)
+	defer module.EventAccountUpdate.Submit(struct{}{})

 	// Create request options.
 	userData := &account.User{}
--- a/spn/access/client_test.go
+++ b/spn/access/client_test.go
@@ -63,7 +63,7 @@ func loginAndRefresh(t *testing.T, doLogin bool, refreshTimes int) {
 		t.Logf("auth token: %+v", authToken.Token)
 	}

-	for i := 0; i < refreshTimes; i++ {
+	for range refreshTimes {
 		user, _, err := UpdateUser()
 		if err != nil {
 			t.Fatalf("getting profile failed: %s", err)
--- a/spn/access/database.go
+++ b/spn/access/database.go
@@ -7,8 +7,8 @@ import (
 	"sync"
 	"time"

-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/database/record"
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/record"
 	"github.com/safing/portmaster/spn/access/account"
 )

--- a/spn/access/module.go
+++ b/spn/access/module.go
@@ -1,25 +1,45 @@
 package access

 import (
-	"context"
 	"errors"
 	"fmt"
+	"sync/atomic"
 	"time"

 	"github.com/tevino/abool"

-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/spn/access/account"
 	"github.com/safing/portmaster/spn/access/token"
 	"github.com/safing/portmaster/spn/conf"
 )

-var (
-	module *modules.Module
+type Access struct {
+	mgr      *mgr.Manager
+	instance instance

-	accountUpdateTask *modules.Task
+	updateAccountWorkerMgr *mgr.WorkerMgr
+
+	EventAccountUpdate *mgr.EventMgr[struct{}]
+}
+
+func (a *Access) Manager() *mgr.Manager {
+	return a.mgr
+}
+
+func (a *Access) Start() error {
+	return start()
+}
+
+func (a *Access) Stop() error {
+	return stop()
+}
+
+var (
+	module     *Access
+	shimLoaded atomic.Bool

 	tokenIssuerIsFailing     = abool.New()
 	tokenIssuerRetryDuration = 10 * time.Minute
@@ -38,15 +58,9 @@ var (
 	ErrNotLoggedIn          = errors.New("not logged in")
 )

-func init() {
-	module = modules.Register("access", prep, start, stop, "terminal")
-}
-
 func prep() error {
-	module.RegisterEvent(AccountUpdateEvent, true)
-
 	// Register API handlers.
-	if conf.Client() {
+	if conf.Integrated() {
 		err := registerAPIEndpoints()
 		if err != nil {
 			return err
@@ -62,25 +76,46 @@ func start() error {
 		return err
 	}

-	if conf.Client() {
+	if conf.Integrated() {
+		// Add config listener to enable/disable SPN.
+		module.instance.Config().EventConfigChange.AddCallback("spn enable check", func(wc *mgr.WorkerCtx, s struct{}) (bool, error) {
+			// Do not do anything when we are shutting down.
+			if module.instance.Stopping() {
+				return true, nil
+			}
+
+			enabled := config.GetAsBool("spn/enable", false)
+			if enabled() {
+				module.mgr.Go("ensure SPN is started", module.instance.SPNGroup().EnsureStartedWorker)
+			} else {
+				module.mgr.Go("ensure SPN is stopped", module.instance.SPNGroup().EnsureStoppedWorker)
+			}
+			return false, nil
+		})
+
+		// Check if we need to enable SPN now.
+		enabled := config.GetAsBool("spn/enable", false)
+		if enabled() {
+			module.mgr.Go("ensure SPN is started", module.instance.SPNGroup().EnsureStartedWorker)
+		}
+
 		// Load tokens from database.
 		loadTokens()

 		// Register new task.
-		accountUpdateTask = module.NewTask(
-			"update account",
-			UpdateAccount,
-		).Repeat(24 * time.Hour).Schedule(time.Now().Add(1 * time.Minute))
+		module.updateAccountWorkerMgr.Delay(1 * time.Minute)
 	}

 	return nil
 }

 func stop() error {
-	if conf.Client() {
-		// Stop account update task.
-		accountUpdateTask.Cancel()
-		accountUpdateTask = nil
+	if conf.Integrated() {
+		// Make sure SPN is stopped before we proceed.
+		err := module.mgr.Do("ensure SPN is shut down", module.instance.SPNGroup().EnsureStoppedWorker)
+		if err != nil {
+			log.Errorf("access: stop SPN: %s", err)
+		}

 		// Store tokens to database.
 		storeTokens()
@@ -93,11 +128,14 @@ func stop() error {
 }

 // UpdateAccount updates the user account and fetches new tokens, if needed.
-func UpdateAccount(_ context.Context, task *modules.Task) error {
+func UpdateAccount(_ *mgr.WorkerCtx) error {
+	// Schedule next call - this will change if other conditions are met bellow.
+	module.updateAccountWorkerMgr.Delay(24 * time.Hour)
+
 	// Retry sooner if the token issuer is failing.
 	defer func() {
-		if tokenIssuerIsFailing.IsSet() && task != nil {
-			task.Schedule(time.Now().Add(tokenIssuerRetryDuration))
+		if tokenIssuerIsFailing.IsSet() {
+			module.updateAccountWorkerMgr.Delay(tokenIssuerRetryDuration)
 		}
 	}()

@@ -128,15 +166,15 @@ func UpdateAccount(_ context.Context, task *modules.Task) error {

 	case time.Until(*u.Subscription.EndsAt) < 24*time.Hour &&
 		time.Since(*u.Subscription.EndsAt) < 24*time.Hour:
-		// Update account every hour 24h hours before and after the subscription ends.
-		task.Schedule(time.Now().Add(time.Hour))
+		// Update account every hour for 24h hours before and after the subscription ends.
+		module.updateAccountWorkerMgr.Delay(1 * time.Hour)

 	case u.Subscription.NextBillingDate == nil: // No auto-subscription.

 	case time.Until(*u.Subscription.NextBillingDate) < 24*time.Hour &&
 		time.Since(*u.Subscription.NextBillingDate) < 24*time.Hour:
 		// Update account every hour 24h hours before and after the next billing date.
-		task.Schedule(time.Now().Add(time.Hour))
+		module.updateAccountWorkerMgr.Delay(1 * time.Hour)
 	}

 	return nil
@@ -165,11 +203,8 @@ func tokenIssuerFailed() {
 	if !tokenIssuerIsFailing.SetToIf(false, true) {
 		return
 	}
-	if !module.Online() {
-		return
-	}

-	accountUpdateTask.Schedule(time.Now().Add(tokenIssuerRetryDuration))
+	module.updateAccountWorkerMgr.Delay(tokenIssuerRetryDuration)
 }

 // IsLoggedIn returns whether a User is currently logged in.
@@ -192,3 +227,31 @@ func (user *UserRecord) MayUseTheSPN() bool {

 	return user.User.MayUseSPN()
 }
+
+// New returns a new Access module.
+func New(instance instance) (*Access, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+
+	m := mgr.New("Access")
+	module = &Access{
+		mgr:      m,
+		instance: instance,
+
+		EventAccountUpdate:     mgr.NewEventMgr[struct{}](AccountUpdateEvent, m),
+		updateAccountWorkerMgr: m.NewWorkerMgr("update account", UpdateAccount, nil),
+	}
+
+	if err := prep(); err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
+
+type instance interface {
+	Config() *config.Config
+	SPNGroup() *mgr.ExtendedGroup
+	Stopping() bool
+}
--- a/spn/access/module_test.go
+++ b/spn/access/module_test.go
@@ -1,13 +1,57 @@
 package access

 import (
+	"fmt"
+	"os"
 	"testing"

-	"github.com/safing/portmaster/service/core/pmtesting"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/spn/conf"
 )

-func TestMain(m *testing.M) {
-	conf.EnableClient(true)
-	pmtesting.TestMain(m, module)
+type testInstance struct {
+	config *config.Config
+}
+
+func (stub *testInstance) Config() *config.Config {
+	return stub.config
+}
+
+func (stub *testInstance) SPNGroup() *mgr.ExtendedGroup {
+	return nil
+}
+
+func (stub *testInstance) Stopping() bool {
+	return false
+}
+func (stub *testInstance) SetCmdLineOperation(f func() error) {}
+
+func TestMain(m *testing.M) {
+	instance := &testInstance{}
+	var err error
+	instance.config, err = config.New(instance)
+	if err != nil {
+		fmt.Printf("failed to create config module: %s", err)
+		os.Exit(0)
+	}
+	module, err = New(instance)
+	if err != nil {
+		fmt.Printf("failed to create access module: %s", err)
+		os.Exit(0)
+	}
+
+	err = instance.config.Start()
+	if err != nil {
+		fmt.Printf("failed to start config module: %s", err)
+		os.Exit(0)
+	}
+	err = module.Start()
+	if err != nil {
+		fmt.Printf("failed to start access module: %s", err)
+		os.Exit(0)
+	}
+
+	conf.EnableClient(true)
+	m.Run()
 }
--- a/spn/access/notify.go
+++ b/spn/access/notify.go
@@ -5,8 +5,8 @@ import (
 	"strings"
 	"time"

-	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/notifications"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/notifications"
 )

 const (
--- a/spn/access/op_auth.go
+++ b/spn/access/op_auth.go
@@ -3,10 +3,10 @@ package access
 import (
 	"time"

-	"github.com/safing/portbase/container"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/access/token"
 	"github.com/safing/portmaster/spn/terminal"
+	"github.com/safing/structures/container"
 )

 // OpTypeAccessCodeAuth is the type ID of the auth operation.
--- a/spn/access/storage.go
+++ b/spn/access/storage.go
@@ -6,12 +6,12 @@ import (
 	"fmt"
 	"time"

-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/database/query"
-	"github.com/safing/portbase/database/record"
-	"github.com/safing/portbase/formats/dsd"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/query"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/access/token"
+	"github.com/safing/structures/dsd"
 )

 func loadTokens() {
@@ -120,7 +120,7 @@ func clearTokens() {
 	}

 	// Purge database storage prefix.
-	ctx, cancel := context.WithTimeout(module.Ctx, 10*time.Second)
+	ctx, cancel := context.WithTimeout(module.mgr.Ctx(), 10*time.Second)
 	defer cancel()
 	n, err := db.Purge(ctx, query.New(fmt.Sprintf(tokenStorageKeyTemplate, "")))
 	if err != nil {
--- a/spn/access/token/module_test.go
+++ b/spn/access/token/module_test.go
@@ -1,13 +1,26 @@
 package token

 import (
+	"fmt"
+	"os"
 	"testing"

-	"github.com/safing/portbase/modules"
-	"github.com/safing/portmaster/service/core/pmtesting"
+	"github.com/safing/portmaster/base/rng"
 )

+type testInstance struct{}
+
 func TestMain(m *testing.M) {
-	module := modules.Register("token", nil, nil, nil, "rng")
-	pmtesting.TestMain(m, module)
+	rng, err := rng.New(testInstance{})
+	if err != nil {
+		fmt.Printf("failed to create RNG module: %s", err)
+		os.Exit(1)
+	}
+
+	err = rng.Start()
+	if err != nil {
+		fmt.Printf("failed to start RNG module: %s", err)
+		os.Exit(1)
+	}
+	m.Run()
 }
--- a/spn/access/token/pblind.go
+++ b/spn/access/token/pblind.go
@@ -13,8 +13,8 @@ import (
 	"github.com/mr-tron/base58"
 	"github.com/rot256/pblind"

-	"github.com/safing/portbase/container"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
 )

 const pblindSecretSize = 32
@@ -217,7 +217,7 @@ func (pbh *PBlindHandler) CreateSetup() (state *PBlindSignerState, setupResponse
 	}

 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		info, err := pbh.makeInfo(i + 1)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create info #%d: %w", i, err)
@@ -257,7 +257,7 @@ func (pbh *PBlindHandler) CreateTokenRequest(requestSetup *PBlindSetupResponse)
 	}

 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Check if we have setup data.
 		if requestSetup.Msgs[i] == nil {
 			return nil, fmt.Errorf("missing setup data #%d", i)
@@ -319,7 +319,7 @@ func (pbh *PBlindHandler) IssueTokens(state *PBlindSignerState, request *PBlindT
 	}

 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Check if we have request data.
 		if request.Msgs[i] == nil {
 			return nil, fmt.Errorf("missing request data #%d", i)
@@ -360,7 +360,7 @@ func (pbh *PBlindHandler) ProcessIssuedTokens(issuedTokens *IssuedPBlindTokens)
 	finalizedTokens := make([]*PBlindToken, pbh.opts.BatchSize)

 	// Go through the batch.
-	for i := 0; i < pbh.opts.BatchSize; i++ {
+	for i := range pbh.opts.BatchSize {
 		// Finalize token.
 		err := pbh.requestState[i].State.ProcessMessage3(*issuedTokens.Msgs[i])
 		if err != nil {
--- a/spn/access/token/pblind_test.go
+++ b/spn/access/token/pblind_test.go
@@ -124,7 +124,7 @@ func TestPBlindLibrary(t *testing.T) {

 	// Create signers and prep requests.
 	start := time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		signer, err := pblind.CreateSigner(sk, info)
 		if err != nil {
 			t.Fatal(err)
@@ -146,7 +146,7 @@ func TestPBlindLibrary(t *testing.T) {

 	// Create requesters and create requests.
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		requester, err := pblind.CreateRequester(pk, info, msgStr)
 		if err != nil {
 			t.Fatal(err)
@@ -178,7 +178,7 @@ func TestPBlindLibrary(t *testing.T) {

 	// Sign requests
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var msg2S pblind.Message2
 		_, err = asn1.Unmarshal(toServer[i], &msg2S)
 		if err != nil {
@@ -204,7 +204,7 @@ func TestPBlindLibrary(t *testing.T) {

 	// Verify signed requests
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var msg3R pblind.Message3
 		_, err := asn1.Unmarshal(toClient[i], &msg3R)
 		if err != nil {
@@ -234,7 +234,7 @@ func TestPBlindLibrary(t *testing.T) {

 	// Verify on server
 	start = time.Now()
-	for i := 0; i < batchSize; i++ {
+	for i := range batchSize {
 		var sig pblind.Signature
 		_, err := asn1.Unmarshal(toServer[i], &sig)
 		if err != nil {
--- a/spn/access/token/request_test.go
+++ b/spn/access/token/request_test.go
@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"

-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/dsd"
 )

 func TestFull(t *testing.T) {
--- a/spn/access/token/scramble.go
+++ b/spn/access/token/scramble.go
@@ -7,7 +7,7 @@ import (
 	"github.com/mr-tron/base58"

 	"github.com/safing/jess/lhash"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/structures/dsd"
 )

 const (
--- a/spn/access/token/token.go
+++ b/spn/access/token/token.go
@@ -8,7 +8,7 @@ import (

 	"github.com/mr-tron/base58"

-	"github.com/safing/portbase/container"
+	"github.com/safing/structures/container"
 )

 // Token represents a token, consisting of a zone (name) and some data.
--- a/spn/access/token/token_test.go
+++ b/spn/access/token/token_test.go
@@ -3,7 +3,7 @@ package token
 import (
 	"testing"

-	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/base/rng"
 )

 func TestToken(t *testing.T) {
--- a/spn/access/zones.go
+++ b/spn/access/zones.go
@@ -9,7 +9,7 @@ import (
 	"github.com/tevino/abool"

 	"github.com/safing/jess/lhash"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/spn/access/token"
 	"github.com/safing/portmaster/spn/conf"
 	"github.com/safing/portmaster/spn/terminal"
@@ -48,7 +48,7 @@ func InitializeZones() error {

 	// Special client zone config.
 	var requestSignalHandler func(token.Handler)
-	if conf.Client() {
+	if conf.Integrated() {
 		requestSignalHandler = shouldRequestTokensHandler
 	}

@@ -139,14 +139,8 @@ func initializeTestZone() error {
 }

 func shouldRequestTokensHandler(_ token.Handler) {
-	// accountUpdateTask is always set in client mode and when the module is online.
-	// Check if it's set in case this gets executed in other circumstances.
-	if accountUpdateTask == nil {
-		log.Warningf("spn/access: trying to trigger account update, but the task is not available")
-		return
-	}
-
-	accountUpdateTask.StartASAP()
+	// Run the account update task as now.
+	module.updateAccountWorkerMgr.Go()
 }

 // GetTokenAmount returns the amount of tokens for the given zones.