Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

base/config/main.go

@@ -0,0 +1,155 @@
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/safing/portmaster/base/dataroot"
+	"github.com/safing/portmaster/base/utils"
+	"github.com/safing/portmaster/base/utils/debug"
+	"github.com/safing/portmaster/service/mgr"
+)
+
+// ChangeEvent is the name of the config change event.
+const ChangeEvent = "config change"
+
+var (
+	dataRoot *utils.DirStructure
+
+	exportConfig bool
+)
+
+// SetDataRoot sets the data root from which the updates module derives its paths.
+func SetDataRoot(root *utils.DirStructure) {
+	if dataRoot == nil {
+		dataRoot = root
+	}
+}
+
+func init() {
+	flag.BoolVar(&exportConfig, "export-config-options", false, "export configuration registry and exit")
+}
+
+func prep() error {
+	SetDataRoot(dataroot.Root())
+	if dataRoot == nil {
+		return errors.New("data root is not set")
+	}
+
+	if exportConfig {
+		module.instance.SetCmdLineOperation(exportConfigCmd)
+		return mgr.ErrExecuteCmdLineOp
+	}
+
+	return registerBasicOptions()
+}
+
+func start() error {
+	configFilePath = filepath.Join(dataRoot.Path, "config.json")
+
+	// Load log level from log package after it started.
+	err := loadLogLevel()
+	if err != nil {
+		return err
+	}
+
+	err = registerAsDatabase()
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	err = loadConfig(false)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("failed to load config file: %w", err)
+	}
+	return nil
+}
+
+func exportConfigCmd() error {
+	// Reset the metrics instance name option, as the default
+	// is set to the current hostname.
+	// Config key copied from metrics.CfgOptionInstanceKey.
+	option, err := GetOption("core/metrics/instance")
+	if err == nil {
+		option.DefaultValue = ""
+	}
+
+	data, err := json.MarshalIndent(ExportOptions(), "", "  ")
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stdout.Write(data)
+	return err
+}
+
+// AddToDebugInfo adds all changed global config options to the given debug.Info.
+func AddToDebugInfo(di *debug.Info) {
+	var lines []string
+
+	// Collect all changed settings.
+	_ = ForEachOption(func(opt *Option) error {
+		opt.Lock()
+		defer opt.Unlock()
+
+		if opt.ReleaseLevel <= getReleaseLevel() && opt.activeValue != nil {
+			if opt.Sensitive {
+				lines = append(lines, fmt.Sprintf("%s: [redacted]", opt.Key))
+			} else {
+				lines = append(lines, fmt.Sprintf("%s: %v", opt.Key, opt.activeValue.getData(opt)))
+			}
+		}
+
+		return nil
+	})
+	sort.Strings(lines)
+
+	// Add data as section.
+	di.AddSection(
+		fmt.Sprintf("Config: %d", len(lines)),
+		debug.UseCodeSection|debug.AddContentLineBreaks,
+		lines...,
+	)
+}
+
+// GetActiveConfigValues returns a map with the active config values.
+func GetActiveConfigValues() map[string]interface{} {
+	values := make(map[string]interface{})
+
+	// Collect active values from options.
+	_ = ForEachOption(func(opt *Option) error {
+		opt.Lock()
+		defer opt.Unlock()
+
+		if opt.ReleaseLevel <= getReleaseLevel() && opt.activeValue != nil {
+			values[opt.Key] = opt.activeValue.getData(opt)
+		}
+
+		return nil
+	})
+
+	return values
+}
+
+// InitializeUnitTestDataroot initializes a new random tmp directory for running tests.
+func InitializeUnitTestDataroot(testName string) (string, error) {
+	basePath, err := os.MkdirTemp("", fmt.Sprintf("portmaster-%s", testName))
+	if err != nil {
+		return "", fmt.Errorf("failed to make tmp dir: %w", err)
+	}
+
+	ds := utils.NewDirStructure(basePath, 0o0755)
+	SetDataRoot(ds)
+	err = dataroot.Initialize(basePath, 0o0755)
+	if err != nil {
+		return "", fmt.Errorf("failed to initialize dataroot: %w", err)
+	}
+
+	return basePath, nil
+}