Restructure modules (#1572)

* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
2024-08-09 17:15:48 +02:00
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
--- a/base/metrics/persistence.go
+++ b/base/metrics/persistence.go
@@ -0,0 +1,153 @@
+package metrics
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/log"
+)
+
+var (
+	storage       *metricsStorage
+	storageKey    string
+	storageInit   = abool.New()
+	storageLoaded = abool.New()
+
+	db = database.NewInterface(&database.Options{
+		Local:    true,
+		Internal: true,
+	})
+
+	// ErrAlreadyInitialized is returned when trying to initialize an option
+	// more than once or if the time window for initializing is over.
+	ErrAlreadyInitialized = errors.New("already initialized")
+)
+
+type metricsStorage struct {
+	sync.Mutex
+	record.Base
+
+	Start    time.Time
+	Counters map[string]uint64
+}
+
+// EnableMetricPersistence enables metric persistence for metrics that opted
+// for it. They given key is the database key where the metric data will be
+// persisted.
+// This call also directly loads the stored data from the database.
+// The returned error is only about loading the metrics, not about enabling
+// persistence.
+// May only be called once.
+func EnableMetricPersistence(key string) error {
+	// Check if already initialized.
+	if !storageInit.SetToIf(false, true) {
+		return ErrAlreadyInitialized
+	}
+
+	// Set storage key.
+	storageKey = key
+
+	// Load metrics from storage.
+	var err error
+	storage, err = getMetricsStorage(storageKey)
+	switch {
+	case err == nil:
+		// Continue.
+	case errors.Is(err, database.ErrNotFound):
+		return nil
+	default:
+		return err
+	}
+	storageLoaded.Set()
+
+	// Load saved state for all counter metrics.
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	for _, m := range registry {
+		counter, ok := m.(*Counter)
+		if ok {
+			counter.loadState()
+		}
+	}
+
+	return nil
+}
+
+func (c *Counter) loadState() {
+	// Check if we can and should load the state.
+	if !storageLoaded.IsSet() || !c.Opts().Persist {
+		return
+	}
+
+	c.Set(storage.Counters[c.LabeledID()])
+}
+
+func storePersistentMetrics() {
+	// Check if persistence is enabled.
+	if !storageInit.IsSet() || storageKey == "" {
+		return
+	}
+
+	// Create new storage.
+	newStorage := &metricsStorage{
+		// TODO: This timestamp should be taken from previous save, if possible.
+		Start:    time.Now(),
+		Counters: make(map[string]uint64),
+	}
+	newStorage.SetKey(storageKey)
+	// Copy values from previous version.
+	if storageLoaded.IsSet() {
+		newStorage.Start = storage.Start
+	}
+
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	// Export all counter metrics.
+	for _, m := range registry {
+		if m.Opts().Persist {
+			counter, ok := m.(*Counter)
+			if ok {
+				newStorage.Counters[m.LabeledID()] = counter.Get()
+			}
+		}
+	}
+
+	// Save to database.
+	err := db.Put(newStorage)
+	if err != nil {
+		log.Warningf("metrics: failed to save metrics storage to db: %s", err)
+	}
+}
+
+func getMetricsStorage(key string) (*metricsStorage, error) {
+	r, err := db.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// unwrap
+	if r.IsWrapped() {
+		// only allocate a new struct, if we need it
+		newStorage := &metricsStorage{}
+		err = record.Unwrap(r, newStorage)
+		if err != nil {
+			return nil, err
+		}
+		return newStorage, nil
+	}
+
+	// or adjust type
+	newStorage, ok := r.(*metricsStorage)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *metricsStorage, but %T", r)
+	}
+	return newStorage, nil
+}