Restructure modules (#1572)

* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
2024-08-09 17:15:48 +02:00
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
--- a/base/metrics/module.go
+++ b/base/metrics/module.go
@@ -0,0 +1,216 @@
+package metrics
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/service/mgr"
+)
+
+type Metrics struct {
+	mgr      *mgr.Manager
+	instance instance
+
+	metricTicker *mgr.SleepyTicker
+}
+
+func (met *Metrics) Manager() *mgr.Manager {
+	return met.mgr
+}
+
+func (met *Metrics) Start() error {
+	return start()
+}
+
+func (met *Metrics) Stop() error {
+	return stop()
+}
+
+func (met *Metrics) SetSleep(enabled bool) {
+	if met.metricTicker != nil {
+		met.metricTicker.SetSleep(enabled)
+	}
+}
+
+var (
+	module     *Metrics
+	shimLoaded atomic.Bool
+
+	registry     []Metric
+	registryLock sync.RWMutex
+
+	readyToRegister       bool
+	firstMetricRegistered bool
+	metricNamespace       string
+	globalLabels          = make(map[string]string)
+
+	// ErrAlreadyStarted is returned when an operation is only valid before the
+	// first metric is registered, and is called after.
+	ErrAlreadyStarted = errors.New("can only be changed before first metric is registered")
+
+	// ErrAlreadyRegistered is returned when a metric with the same ID is
+	// registered again.
+	ErrAlreadyRegistered = errors.New("metric already registered")
+
+	// ErrAlreadySet is returned when a value is already set and cannot be changed.
+	ErrAlreadySet = errors.New("already set")
+
+	// ErrInvalidOptions is returned when invalid options where provided.
+	ErrInvalidOptions = errors.New("invalid options")
+)
+
+func start() error {
+	// Add metric instance name as global variable if set.
+	if instanceOption() != "" {
+		if err := AddGlobalLabel("instance", instanceOption()); err != nil {
+			return err
+		}
+	}
+
+	// Mark registry as ready to register metrics.
+	func() {
+		registryLock.Lock()
+		defer registryLock.Unlock()
+		readyToRegister = true
+	}()
+
+	if err := registerInfoMetric(); err != nil {
+		return err
+	}
+
+	if err := registerRuntimeMetric(); err != nil {
+		return err
+	}
+
+	if err := registerHostMetrics(); err != nil {
+		return err
+	}
+
+	if err := registerLogMetrics(); err != nil {
+		return err
+	}
+
+	if pushOption() != "" {
+		module.mgr.Go("metric pusher", metricsWriter)
+	}
+
+	return nil
+}
+
+func stop() error {
+	// Wait until the metrics pusher is done, as it may have started reporting
+	// and may report a higher number than we store to disk. For persistent
+	// metrics it can then happen that the first report is lower than the
+	// previous report, making prometheus think that all that happened since the
+	// last report, due to the automatic restart detection.
+
+	// The registry is read locked when writing metrics.
+	// Write lock the registry to make sure all writes are finished.
+	registryLock.Lock()
+	registryLock.Unlock() //nolint:staticcheck
+
+	storePersistentMetrics()
+
+	return nil
+}
+
+func register(m Metric) error {
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	// Check if metric ID is already registered.
+	for _, registeredMetric := range registry {
+		if m.LabeledID() == registeredMetric.LabeledID() {
+			return ErrAlreadyRegistered
+		}
+		if m.Opts().InternalID != "" &&
+			m.Opts().InternalID == registeredMetric.Opts().InternalID {
+			return fmt.Errorf("%w with this internal ID", ErrAlreadyRegistered)
+		}
+	}
+
+	// Add new metric to registry and sort it.
+	registry = append(registry, m)
+	sort.Sort(byLabeledID(registry))
+
+	// Check if we can already register.
+	if !readyToRegister {
+		return fmt.Errorf("registering metric %q too early", m.ID())
+	}
+
+	// Set flag that first metric is now registered.
+	firstMetricRegistered = true
+
+	return nil
+}
+
+// SetNamespace sets the namespace for all metrics. It is prefixed to all
+// metric IDs.
+// It must be set before any metric is registered.
+// Does not affect golang runtime metrics.
+func SetNamespace(namespace string) error {
+	// Lock registry and check if a first metric is already registered.
+	registryLock.Lock()
+	defer registryLock.Unlock()
+	if firstMetricRegistered {
+		return ErrAlreadyStarted
+	}
+
+	// Check if the namespace is already set.
+	if metricNamespace != "" {
+		return ErrAlreadySet
+	}
+
+	metricNamespace = namespace
+	return nil
+}
+
+// AddGlobalLabel adds a global label to all metrics.
+// Global labels must be added before any metric is registered.
+// Does not affect golang runtime metrics.
+func AddGlobalLabel(name, value string) error {
+	// Lock registry and check if a first metric is already registered.
+	registryLock.Lock()
+	defer registryLock.Unlock()
+	if firstMetricRegistered {
+		return ErrAlreadyStarted
+	}
+
+	// Check format.
+	if !prometheusFormat.MatchString(name) {
+		return fmt.Errorf("metric label name %q must match %s", name, PrometheusFormatRequirement)
+	}
+
+	globalLabels[name] = value
+	return nil
+}
+
+type byLabeledID []Metric
+
+func (r byLabeledID) Len() int           { return len(r) }
+func (r byLabeledID) Less(i, j int) bool { return r[i].LabeledID() < r[j].LabeledID() }
+func (r byLabeledID) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+
+func New(instance instance) (*Metrics, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Metrics")
+	module = &Metrics{
+		mgr:      m,
+		instance: instance,
+	}
+	if err := prepConfig(); err != nil {
+		return nil, err
+	}
+
+	if err := registerAPI(); err != nil {
+		return nil, err
+	}
+	return module, nil
+}
+
+type instance interface{}