Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

service/sync/module.go

@@ -1,23 +1,35 @@
 package sync
 
 import (
-	"github.com/safing/portbase/database"
-	"github.com/safing/portbase/modules"
+	"errors"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/service/mgr"
 )
 
-var (
-	module *modules.Module
-
-	db = database.NewInterface(&database.Options{
-		Local:    true,
-		Internal: true,
-	})
-)
-
-func init() {
-	module = modules.Register("sync", prep, nil, nil, "profiles")
+type Sync struct {
+	mgr      *mgr.Manager
+	instance instance
 }
 
+func (s *Sync) Manager() *mgr.Manager {
+	return s.mgr
+}
+
+func (s *Sync) Start() error {
+	return nil
+}
+
+func (s *Sync) Stop() error {
+	return nil
+}
+
+var db = database.NewInterface(&database.Options{
+	Local:    true,
+	Internal: true,
+})
+
 func prep() error {
 	if err := registerSettingsAPI(); err != nil {
 		return err
@@ -30,3 +42,28 @@ func prep() error {
 	}
 	return nil
 }
+
+var (
+	module     *Sync
+	shimLoaded atomic.Bool
+)
+
+// New returns a new NetEnv module.
+func New(instance instance) (*Sync, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Sync")
+	module = &Sync{
+		mgr:      m,
+		instance: instance,
+	}
+
+	if err := prep(); err != nil {
+		return nil, err
+	}
+
+	return module, nil
+}
+
+type instance interface{}

service/sync/profile.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/vincent-petithory/dataurl"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/service/profile"
 	"github.com/safing/portmaster/service/profile/binmeta"
 )
@@ -97,8 +97,7 @@ func registerProfileAPI() error {
 			Field:       "id",
 			Description: "Specify scoped profile ID to export.",
 		}},
-		BelongsTo: module,
-		DataFunc:  handleExportProfile,
+		DataFunc: handleExportProfile,
 	}); err != nil {
 		return err
 	}
@@ -128,7 +127,6 @@ func registerProfileAPI() error {
 				Description: "Allow importing of unknown values.",
 			},
 		},
-		BelongsTo:  module,
 		StructFunc: handleImportProfile,
 	}); err != nil {
 		return err

service/sync/setting_single.go

@@ -7,10 +7,10 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/config"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
 	"github.com/safing/portmaster/service/profile"
+	"github.com/safing/structures/dsd"
 )
 
 // SingleSettingExport holds an export of a single setting.
@@ -44,8 +44,7 @@ func registerSingleSettingAPI() error {
 			Field:       "key",
 			Description: "Specify which settings key to export.",
 		}},
-		BelongsTo: module,
-		DataFunc:  handleExportSingleSetting,
+		DataFunc: handleExportSingleSetting,
 	}); err != nil {
 		return err
 	}
@@ -69,7 +68,6 @@ func registerSingleSettingAPI() error {
 			Field:       "validate",
 			Description: "Validate only.",
 		}},
-		BelongsTo:  module,
 		StructFunc: handleImportSingleSetting,
 	}); err != nil {
 		return err

service/sync/settings.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/config"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
 	"github.com/safing/portmaster/service/profile"
 )
 
@@ -52,8 +52,7 @@ func registerSettingsAPI() error {
 			Field:       "key",
 			Description: "Optionally select a single setting to export. Repeat to export selection.",
 		}},
-		BelongsTo: module,
-		DataFunc:  handleExportSettings,
+		DataFunc: handleExportSettings,
 	}); err != nil {
 		return err
 	}
@@ -81,7 +80,6 @@ func registerSettingsAPI() error {
 			Field:       "allowUnknown",
 			Description: "Allow importing of unknown values.",
 		}},
-		BelongsTo:  module,
 		StructFunc: handleImportSettings,
 	}); err != nil {
 		return err
@@ -156,7 +154,7 @@ func handleImportSettings(ar *api.Request) (any, error) {
 		}
 		request.Export = export
 	case request.Export != nil:
-		// Export is aleady parsed.
+		// Export is already parsed.
 	default:
 		return nil, ErrInvalidImportRequest
 	}

service/sync/util.go

@@ -9,9 +9,9 @@ import (
 	yaml "gopkg.in/yaml.v3"
 
 	"github.com/safing/jess/filesig"
-	"github.com/safing/portbase/api"
-	"github.com/safing/portbase/container"
-	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
 )
 
 // Type is the type of an export.