Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

base/metrics/api.go

@@ -0,0 +1,157 @@
+package metrics
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/mgr"
+)
+
+func registerAPI() error {
+	api.RegisterHandler("/metrics", &metricsAPI{})
+
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Export Registered Metrics",
+		Description: "List all registered metrics with their metadata.",
+		Path:        "metrics/list",
+		Read:        api.Dynamic,
+		StructFunc: func(ar *api.Request) (any, error) {
+			return ExportMetrics(ar.AuthToken.Read), nil
+		},
+	}); err != nil {
+		return err
+	}
+
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Export Metric Values",
+		Description: "List all exportable metric values.",
+		Path:        "metrics/values",
+		Read:        api.Dynamic,
+		Parameters: []api.Parameter{{
+			Method:      http.MethodGet,
+			Field:       "internal-only",
+			Description: "Specify to only return metrics with an alternative internal ID.",
+		}},
+		StructFunc: func(ar *api.Request) (any, error) {
+			return ExportValues(
+				ar.AuthToken.Read,
+				ar.Request.URL.Query().Has("internal-only"),
+			), nil
+		},
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type metricsAPI struct{}
+
+func (m *metricsAPI) ReadPermission(*http.Request) api.Permission { return api.Dynamic }
+
+func (m *metricsAPI) WritePermission(*http.Request) api.Permission { return api.NotSupported }
+
+func (m *metricsAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Get API Request for permission and query.
+	ar := api.GetAPIRequest(r)
+	if ar == nil {
+		http.Error(w, "Missing API Request.", http.StatusInternalServerError)
+		return
+	}
+
+	// Get expertise level from query.
+	expertiseLevel := config.ExpertiseLevelDeveloper
+	switch ar.Request.URL.Query().Get("level") {
+	case config.ExpertiseLevelNameUser:
+		expertiseLevel = config.ExpertiseLevelUser
+	case config.ExpertiseLevelNameExpert:
+		expertiseLevel = config.ExpertiseLevelExpert
+	case config.ExpertiseLevelNameDeveloper:
+		expertiseLevel = config.ExpertiseLevelDeveloper
+	}
+
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	w.WriteHeader(http.StatusOK)
+	WriteMetrics(w, ar.AuthToken.Read, expertiseLevel)
+}
+
+// WriteMetrics writes all metrics that match the given permission and
+// expertiseLevel to the given writer.
+func WriteMetrics(w io.Writer, permission api.Permission, expertiseLevel config.ExpertiseLevel) {
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	// Write all matching metrics.
+	for _, metric := range registry {
+		if permission >= metric.Opts().Permission &&
+			expertiseLevel >= metric.Opts().ExpertiseLevel {
+			metric.WritePrometheus(w)
+		}
+	}
+}
+
+func writeMetricsTo(ctx context.Context, url string) error {
+	// First, collect metrics into buffer.
+	buf := &bytes.Buffer{}
+	WriteMetrics(buf, api.PermitSelf, config.ExpertiseLevelDeveloper)
+
+	// Check if there is something to send.
+	if buf.Len() == 0 {
+		log.Debugf("metrics: not pushing metrics, nothing to send")
+		return nil
+	}
+
+	// Create request
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, buf)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Send.
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	// Check return status.
+	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
+		return nil
+	}
+
+	// Get and return error.
+	body, _ := io.ReadAll(resp.Body)
+	return fmt.Errorf(
+		"got %s while writing metrics to %s: %s",
+		resp.Status,
+		url,
+		body,
+	)
+}
+
+func metricsWriter(ctx *mgr.WorkerCtx) error {
+	pushURL := pushOption()
+	module.metricTicker = mgr.NewSleepyTicker(1*time.Minute, 0)
+	defer module.metricTicker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-module.metricTicker.Wait():
+			err := writeMetricsTo(ctx.Ctx(), pushURL)
+			if err != nil {
+				return err
+			}
+		}
+	}
+}

base/metrics/config.go

@@ -0,0 +1,108 @@
+package metrics
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/safing/portmaster/base/config"
+)
+
+// Configuration Keys.
+var (
+	CfgOptionInstanceKey   = "core/metrics/instance"
+	instanceOption         config.StringOption
+	cfgOptionInstanceOrder = 0
+
+	CfgOptionCommentKey   = "core/metrics/comment"
+	commentOption         config.StringOption
+	cfgOptionCommentOrder = 0
+
+	CfgOptionPushKey   = "core/metrics/push"
+	pushOption         config.StringOption
+	cfgOptionPushOrder = 0
+
+	instanceFlag    string
+	defaultInstance string
+	commentFlag     string
+	pushFlag        string
+)
+
+func init() {
+	hostname, err := os.Hostname()
+	if err == nil {
+		hostname = strings.ReplaceAll(hostname, "-", "")
+		if prometheusFormat.MatchString(hostname) {
+			defaultInstance = hostname
+		}
+	}
+
+	flag.StringVar(&instanceFlag, "metrics-instance", defaultInstance, "set the default metrics instance label for all metrics")
+	flag.StringVar(&commentFlag, "metrics-comment", "", "set the default metrics comment label")
+	flag.StringVar(&pushFlag, "push-metrics", "", "set default URL to push prometheus metrics to")
+}
+
+func prepConfig() error {
+	err := config.Register(&config.Option{
+		Name:            "Metrics Instance Name",
+		Key:             CfgOptionInstanceKey,
+		Description:     "Define the prometheus instance label for all exported metrics. Please note that changing the metrics instance name will reset persisted metrics.",
+		Sensitive:       true,
+		OptType:         config.OptTypeString,
+		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ReleaseLevel:    config.ReleaseLevelStable,
+		DefaultValue:    instanceFlag,
+		RequiresRestart: true,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionInstanceOrder,
+			config.CategoryAnnotation:     "Metrics",
+		},
+		ValidationRegex: "^(" + prometheusBaseFormt + ")?$",
+	})
+	if err != nil {
+		return err
+	}
+	instanceOption = config.Concurrent.GetAsString(CfgOptionInstanceKey, instanceFlag)
+
+	err = config.Register(&config.Option{
+		Name:            "Metrics Comment Label",
+		Key:             CfgOptionCommentKey,
+		Description:     "Define a metrics comment label, which is added to the info metric.",
+		Sensitive:       true,
+		OptType:         config.OptTypeString,
+		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ReleaseLevel:    config.ReleaseLevelStable,
+		DefaultValue:    commentFlag,
+		RequiresRestart: true,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionCommentOrder,
+			config.CategoryAnnotation:     "Metrics",
+		},
+	})
+	if err != nil {
+		return err
+	}
+	commentOption = config.Concurrent.GetAsString(CfgOptionCommentKey, commentFlag)
+
+	err = config.Register(&config.Option{
+		Name:            "Push Prometheus Metrics",
+		Key:             CfgOptionPushKey,
+		Description:     "Push metrics to this URL in the prometheus format.",
+		Sensitive:       true,
+		OptType:         config.OptTypeString,
+		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ReleaseLevel:    config.ReleaseLevelStable,
+		DefaultValue:    pushFlag,
+		RequiresRestart: true,
+		Annotations: config.Annotations{
+			config.DisplayOrderAnnotation: cfgOptionPushOrder,
+			config.CategoryAnnotation:     "Metrics",
+		},
+	})
+	if err != nil {
+		return err
+	}
+	pushOption = config.Concurrent.GetAsString(CfgOptionPushKey, pushFlag)
+
+	return nil
+}

base/metrics/metric.go

@@ -0,0 +1,165 @@
+package metrics
+
+import (
+	"fmt"
+	"io"
+	"regexp"
+	"sort"
+	"strings"
+
+	vm "github.com/VictoriaMetrics/metrics"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+)
+
+// PrometheusFormatRequirement is required format defined by prometheus for
+// metric and label names.
+const (
+	prometheusBaseFormt         = "[a-zA-Z_][a-zA-Z0-9_]*"
+	PrometheusFormatRequirement = "^" + prometheusBaseFormt + "$"
+)
+
+var prometheusFormat = regexp.MustCompile(PrometheusFormatRequirement)
+
+// Metric represents one or more metrics.
+type Metric interface {
+	ID() string
+	LabeledID() string
+	Opts() *Options
+	WritePrometheus(w io.Writer)
+}
+
+type metricBase struct {
+	Identifier        string
+	Labels            map[string]string
+	LabeledIdentifier string
+	Options           *Options
+	set               *vm.Set
+}
+
+// Options can be used to set advanced metric settings.
+type Options struct {
+	// Name defines an optional human readable name for the metric.
+	Name string
+
+	// InternalID specifies an alternative internal ID that will be used when
+	// exposing the metric via the API in a structured format.
+	InternalID string
+
+	// AlertLimit defines an upper limit that triggers an alert.
+	AlertLimit float64
+
+	// AlertTimeframe defines an optional timeframe in seconds for which the
+	// AlertLimit should be interpreted in.
+	AlertTimeframe float64
+
+	// Permission defines the permission that is required to read the metric.
+	Permission api.Permission
+
+	// ExpertiseLevel defines the expertise level that the metric is meant for.
+	ExpertiseLevel config.ExpertiseLevel
+
+	// Persist enabled persisting the metric on shutdown and loading the previous
+	// value at start. This is only supported for counters.
+	Persist bool
+}
+
+func newMetricBase(id string, labels map[string]string, opts Options) (*metricBase, error) {
+	// Check formats.
+	if !prometheusFormat.MatchString(strings.ReplaceAll(id, "/", "_")) {
+		return nil, fmt.Errorf("metric name %q must match %s", id, PrometheusFormatRequirement)
+	}
+	for labelName := range labels {
+		if !prometheusFormat.MatchString(labelName) {
+			return nil, fmt.Errorf("metric label name %q must match %s", labelName, PrometheusFormatRequirement)
+		}
+	}
+
+	// Check permission.
+	if opts.Permission < api.PermitAnyone {
+		// Default to PermitUser.
+		opts.Permission = api.PermitUser
+	}
+
+	// Ensure that labels is a map.
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+
+	// Create metric base.
+	base := &metricBase{
+		Identifier: id,
+		Labels:     labels,
+		Options:    &opts,
+		set:        vm.NewSet(),
+	}
+	base.LabeledIdentifier = base.buildLabeledID()
+	return base, nil
+}
+
+// ID returns the given ID of the metric.
+func (m *metricBase) ID() string {
+	return m.Identifier
+}
+
+// LabeledID returns the Prometheus-compatible labeled ID of the metric.
+func (m *metricBase) LabeledID() string {
+	return m.LabeledIdentifier
+}
+
+// Opts returns the metric options. They  may not be modified.
+func (m *metricBase) Opts() *Options {
+	return m.Options
+}
+
+// WritePrometheus writes the metric in the prometheus format to the given writer.
+func (m *metricBase) WritePrometheus(w io.Writer) {
+	m.set.WritePrometheus(w)
+}
+
+func (m *metricBase) buildLabeledID() string {
+	// Because we use the namespace and the global flags here, we need to flag
+	// them as immutable.
+	registryLock.Lock()
+	defer registryLock.Unlock()
+	firstMetricRegistered = true
+
+	// Build ID from Identifier.
+	metricID := strings.TrimSpace(strings.ReplaceAll(m.Identifier, "/", "_"))
+
+	// Add namespace to ID.
+	if metricNamespace != "" {
+		metricID = metricNamespace + "_" + metricID
+	}
+
+	// Return now if no labels are defined.
+	if len(globalLabels) == 0 && len(m.Labels) == 0 {
+		return metricID
+	}
+
+	// Add global labels to the custom ones, if they don't exist yet.
+	for labelName, labelValue := range globalLabels {
+		if _, ok := m.Labels[labelName]; !ok {
+			m.Labels[labelName] = labelValue
+		}
+	}
+
+	// Render labels into a slice and sort them in order to make the labeled ID
+	// reproducible.
+	labels := make([]string, 0, len(m.Labels))
+	for labelName, labelValue := range m.Labels {
+		labels = append(labels, fmt.Sprintf("%s=%q", labelName, labelValue))
+	}
+	sort.Strings(labels)
+
+	// Return fully labaled ID.
+	return fmt.Sprintf("%s{%s}", metricID, strings.Join(labels, ","))
+}
+
+// Split metrics into sets, according to the API Auth Levels, which will also correspond to the UI Mode levels. SPN // nodes will also allow public access to metrics with the permission "PermitAnyone".
+// Save "life-long" metrics on shutdown and load them at start.
+// Generate the correct metric name and labels.
+// Expose metrics via http, but also via the runtime DB in order to push metrics to the UI.
+// The UI will have to parse the prometheus metrics format and will not be able to immediately present historical data, // but data will have to be built.
+// Provide the option to push metrics to a prometheus push gateway, this is especially helpful when gathering data from // loads of SPN nodes.

base/metrics/metric_counter.go

@@ -0,0 +1,49 @@
+package metrics
+
+import (
+	vm "github.com/VictoriaMetrics/metrics"
+)
+
+// Counter is a counter metric.
+type Counter struct {
+	*metricBase
+	*vm.Counter
+}
+
+// NewCounter registers a new counter metric.
+func NewCounter(id string, labels map[string]string, opts *Options) (*Counter, error) {
+	// Ensure that there are options.
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	// Make base.
+	base, err := newMetricBase(id, labels, *opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create metric struct.
+	m := &Counter{
+		metricBase: base,
+	}
+
+	// Create metric in set
+	m.Counter = m.set.NewCounter(m.LabeledID())
+
+	// Register metric.
+	err = register(m)
+	if err != nil {
+		return nil, err
+	}
+
+	// Load state.
+	m.loadState()
+
+	return m, nil
+}
+
+// CurrentValue returns the current counter value.
+func (c *Counter) CurrentValue() uint64 {
+	return c.Get()
+}

base/metrics/metric_counter_fetching.go

@@ -0,0 +1,62 @@
+package metrics
+
+import (
+	"fmt"
+	"io"
+
+	vm "github.com/VictoriaMetrics/metrics"
+)
+
+// FetchingCounter is a counter metric that fetches the values via a function call.
+type FetchingCounter struct {
+	*metricBase
+	counter  *vm.Counter
+	fetchCnt func() uint64
+}
+
+// NewFetchingCounter registers a new fetching counter metric.
+func NewFetchingCounter(id string, labels map[string]string, fn func() uint64, opts *Options) (*FetchingCounter, error) {
+	// Check if a fetch function is provided.
+	if fn == nil {
+		return nil, fmt.Errorf("%w: no fetch function provided", ErrInvalidOptions)
+	}
+
+	// Ensure that there are options.
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	// Make base.
+	base, err := newMetricBase(id, labels, *opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create metric struct.
+	m := &FetchingCounter{
+		metricBase: base,
+		fetchCnt:   fn,
+	}
+
+	// Create metric in set
+	m.counter = m.set.NewCounter(m.LabeledID())
+
+	// Register metric.
+	err = register(m)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// CurrentValue returns the current counter value.
+func (fc *FetchingCounter) CurrentValue() uint64 {
+	return fc.fetchCnt()
+}
+
+// WritePrometheus writes the metric in the prometheus format to the given writer.
+func (fc *FetchingCounter) WritePrometheus(w io.Writer) {
+	fc.counter.Set(fc.fetchCnt())
+	fc.metricBase.set.WritePrometheus(w)
+}

base/metrics/metric_export.go

@@ -0,0 +1,89 @@
+package metrics
+
+import (
+	"github.com/safing/portmaster/base/api"
+)
+
+// UIntMetric is an interface for special functions of uint metrics.
+type UIntMetric interface {
+	CurrentValue() uint64
+}
+
+// FloatMetric is an interface for special functions of float metrics.
+type FloatMetric interface {
+	CurrentValue() float64
+}
+
+// MetricExport is used to export a metric and its current value.
+type MetricExport struct {
+	Metric
+	CurrentValue any
+}
+
+// ExportMetrics exports all registered metrics.
+func ExportMetrics(requestPermission api.Permission) []*MetricExport {
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	export := make([]*MetricExport, 0, len(registry))
+	for _, metric := range registry {
+		// Check permission.
+		if requestPermission < metric.Opts().Permission {
+			continue
+		}
+
+		// Add metric with current value.
+		export = append(export, &MetricExport{
+			Metric:       metric,
+			CurrentValue: getCurrentValue(metric),
+		})
+	}
+
+	return export
+}
+
+// ExportValues exports the values of all supported metrics.
+func ExportValues(requestPermission api.Permission, internalOnly bool) map[string]any {
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	export := make(map[string]any, len(registry))
+	for _, metric := range registry {
+		// Check permission.
+		if requestPermission < metric.Opts().Permission {
+			continue
+		}
+
+		// Get Value.
+		v := getCurrentValue(metric)
+		if v == nil {
+			continue
+		}
+
+		// Get ID.
+		var id string
+		switch {
+		case metric.Opts().InternalID != "":
+			id = metric.Opts().InternalID
+		case internalOnly:
+			continue
+		default:
+			id = metric.LabeledID()
+		}
+
+		// Add to export
+		export[id] = v
+	}
+
+	return export
+}
+
+func getCurrentValue(metric Metric) any {
+	if m, ok := metric.(UIntMetric); ok {
+		return m.CurrentValue()
+	}
+	if m, ok := metric.(FloatMetric); ok {
+		return m.CurrentValue()
+	}
+	return nil
+}

base/metrics/metric_gauge.go

@@ -0,0 +1,46 @@
+package metrics
+
+import (
+	vm "github.com/VictoriaMetrics/metrics"
+)
+
+// Gauge is a gauge metric.
+type Gauge struct {
+	*metricBase
+	*vm.Gauge
+}
+
+// NewGauge registers a new gauge metric.
+func NewGauge(id string, labels map[string]string, fn func() float64, opts *Options) (*Gauge, error) {
+	// Ensure that there are options.
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	// Make base.
+	base, err := newMetricBase(id, labels, *opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create metric struct.
+	m := &Gauge{
+		metricBase: base,
+	}
+
+	// Create metric in set
+	m.Gauge = m.set.NewGauge(m.LabeledID(), fn)
+
+	// Register metric.
+	err = register(m)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+// CurrentValue returns the current gauge value.
+func (g *Gauge) CurrentValue() float64 {
+	return g.Get()
+}

base/metrics/metric_histogram.go

@@ -0,0 +1,41 @@
+package metrics
+
+import (
+	vm "github.com/VictoriaMetrics/metrics"
+)
+
+// Histogram is a histogram metric.
+type Histogram struct {
+	*metricBase
+	*vm.Histogram
+}
+
+// NewHistogram registers a new histogram metric.
+func NewHistogram(id string, labels map[string]string, opts *Options) (*Histogram, error) {
+	// Ensure that there are options.
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	// Make base.
+	base, err := newMetricBase(id, labels, *opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create metric struct.
+	m := &Histogram{
+		metricBase: base,
+	}
+
+	// Create metric in set
+	m.Histogram = m.set.NewHistogram(m.LabeledID())
+
+	// Register metric.
+	err = register(m)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}

base/metrics/metrics_host.go

@@ -0,0 +1,263 @@
+package metrics
+
+import (
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/load"
+	"github.com/shirou/gopsutil/mem"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/dataroot"
+	"github.com/safing/portmaster/base/log"
+)
+
+const hostStatTTL = 1 * time.Second
+
+func registerHostMetrics() (err error) {
+	// Register load average metrics.
+	_, err = NewGauge("host/load/avg/1", nil, getFloat64HostStat(LoadAvg1), &Options{Name: "Host Load Avg 1min", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/load/avg/5", nil, getFloat64HostStat(LoadAvg5), &Options{Name: "Host Load Avg 5min", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/load/avg/15", nil, getFloat64HostStat(LoadAvg15), &Options{Name: "Host Load Avg 15min", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+
+	// Register memory usage metrics.
+	_, err = NewGauge("host/mem/total", nil, getUint64HostStat(MemTotal), &Options{Name: "Host Memory Total", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/mem/used", nil, getUint64HostStat(MemUsed), &Options{Name: "Host Memory Used", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/mem/available", nil, getUint64HostStat(MemAvailable), &Options{Name: "Host Memory Available", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/mem/used/percent", nil, getFloat64HostStat(MemUsedPercent), &Options{Name: "Host Memory Used in Percent", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+
+	// Register disk usage metrics.
+	_, err = NewGauge("host/disk/total", nil, getUint64HostStat(DiskTotal), &Options{Name: "Host Disk Total", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/disk/used", nil, getUint64HostStat(DiskUsed), &Options{Name: "Host Disk Used", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/disk/free", nil, getUint64HostStat(DiskFree), &Options{Name: "Host Disk Free", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+	_, err = NewGauge("host/disk/used/percent", nil, getFloat64HostStat(DiskUsedPercent), &Options{Name: "Host Disk Used in Percent", Permission: api.PermitUser})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getUint64HostStat(getStat func() (uint64, bool)) func() float64 {
+	return func() float64 {
+		val, _ := getStat()
+		return float64(val)
+	}
+}
+
+func getFloat64HostStat(getStat func() (float64, bool)) func() float64 {
+	return func() float64 {
+		val, _ := getStat()
+		return val
+	}
+}
+
+var (
+	loadAvg        *load.AvgStat
+	loadAvgExpires time.Time
+	loadAvgLock    sync.Mutex
+)
+
+func getLoadAvg() *load.AvgStat {
+	loadAvgLock.Lock()
+	defer loadAvgLock.Unlock()
+
+	// Return cache if still valid.
+	if time.Now().Before(loadAvgExpires) {
+		return loadAvg
+	}
+
+	// Refresh.
+	var err error
+	loadAvg, err = load.Avg()
+	if err != nil {
+		log.Warningf("metrics: failed to get load avg: %s", err)
+		loadAvg = nil
+	}
+	loadAvgExpires = time.Now().Add(hostStatTTL)
+
+	return loadAvg
+}
+
+// LoadAvg1 returns the 1-minute average system load.
+func LoadAvg1() (loadAvg float64, ok bool) {
+	if stat := getLoadAvg(); stat != nil {
+		return stat.Load1 / float64(runtime.NumCPU()), true
+	}
+	return 0, false
+}
+
+// LoadAvg5 returns the 5-minute average system load.
+func LoadAvg5() (loadAvg float64, ok bool) {
+	if stat := getLoadAvg(); stat != nil {
+		return stat.Load5 / float64(runtime.NumCPU()), true
+	}
+	return 0, false
+}
+
+// LoadAvg15 returns the 15-minute average system load.
+func LoadAvg15() (loadAvg float64, ok bool) {
+	if stat := getLoadAvg(); stat != nil {
+		return stat.Load15 / float64(runtime.NumCPU()), true
+	}
+	return 0, false
+}
+
+var (
+	memStat        *mem.VirtualMemoryStat
+	memStatExpires time.Time
+	memStatLock    sync.Mutex
+)
+
+func getMemStat() *mem.VirtualMemoryStat {
+	memStatLock.Lock()
+	defer memStatLock.Unlock()
+
+	// Return cache if still valid.
+	if time.Now().Before(memStatExpires) {
+		return memStat
+	}
+
+	// Refresh.
+	var err error
+	memStat, err = mem.VirtualMemory()
+	if err != nil {
+		log.Warningf("metrics: failed to get load avg: %s", err)
+		memStat = nil
+	}
+	memStatExpires = time.Now().Add(hostStatTTL)
+
+	return memStat
+}
+
+// MemTotal returns the total system memory.
+func MemTotal() (total uint64, ok bool) {
+	if stat := getMemStat(); stat != nil {
+		return stat.Total, true
+	}
+	return 0, false
+}
+
+// MemUsed returns the used system memory.
+func MemUsed() (used uint64, ok bool) {
+	if stat := getMemStat(); stat != nil {
+		return stat.Used, true
+	}
+	return 0, false
+}
+
+// MemAvailable returns the available system memory.
+func MemAvailable() (available uint64, ok bool) {
+	if stat := getMemStat(); stat != nil {
+		return stat.Available, true
+	}
+	return 0, false
+}
+
+// MemUsedPercent returns the percent of used system memory.
+func MemUsedPercent() (usedPercent float64, ok bool) {
+	if stat := getMemStat(); stat != nil {
+		return stat.UsedPercent, true
+	}
+	return 0, false
+}
+
+var (
+	diskStat        *disk.UsageStat
+	diskStatExpires time.Time
+	diskStatLock    sync.Mutex
+)
+
+func getDiskStat() *disk.UsageStat {
+	diskStatLock.Lock()
+	defer diskStatLock.Unlock()
+
+	// Return cache if still valid.
+	if time.Now().Before(diskStatExpires) {
+		return diskStat
+	}
+
+	// Check if we have a data root.
+	dataRoot := dataroot.Root()
+	if dataRoot == nil {
+		log.Warning("metrics: cannot get disk stats without data root")
+		diskStat = nil
+		diskStatExpires = time.Now().Add(hostStatTTL)
+		return diskStat
+	}
+
+	// Refresh.
+	var err error
+	diskStat, err = disk.Usage(dataRoot.Path)
+	if err != nil {
+		log.Warningf("metrics: failed to get load avg: %s", err)
+		diskStat = nil
+	}
+	diskStatExpires = time.Now().Add(hostStatTTL)
+
+	return diskStat
+}
+
+// DiskTotal returns the total disk space (from the program's data root).
+func DiskTotal() (total uint64, ok bool) {
+	if stat := getDiskStat(); stat != nil {
+		return stat.Total, true
+	}
+	return 0, false
+}
+
+// DiskUsed returns the used disk space (from the program's data root).
+func DiskUsed() (used uint64, ok bool) {
+	if stat := getDiskStat(); stat != nil {
+		return stat.Used, true
+	}
+	return 0, false
+}
+
+// DiskFree returns the available disk space (from the program's data root).
+func DiskFree() (free uint64, ok bool) {
+	if stat := getDiskStat(); stat != nil {
+		return stat.Free, true
+	}
+	return 0, false
+}
+
+// DiskUsedPercent returns the percent of used disk space (from the program's data root).
+func DiskUsedPercent() (usedPercent float64, ok bool) {
+	if stat := getDiskStat(); stat != nil {
+		return stat.UsedPercent, true
+	}
+	return 0, false
+}

base/metrics/metrics_info.go

@@ -0,0 +1,45 @@
+package metrics
+
+import (
+	"runtime"
+	"strings"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/base/info"
+)
+
+var reportedStart atomic.Bool
+
+func registerInfoMetric() error {
+	meta := info.GetInfo()
+	_, err := NewGauge(
+		"info",
+		map[string]string{
+			"version":      checkUnknown(meta.Version),
+			"commit":       checkUnknown(meta.Commit),
+			"build_date":   checkUnknown(meta.BuildTime),
+			"build_source": checkUnknown(meta.Source),
+			"go_os":        runtime.GOOS,
+			"go_arch":      runtime.GOARCH,
+			"go_version":   runtime.Version(),
+			"go_compiler":  runtime.Compiler,
+			"comment":      commentOption(),
+		},
+		func() float64 {
+			// Report as 0 the first time in order to detect (re)starts.
+			if reportedStart.CompareAndSwap(false, true) {
+				return 0
+			}
+			return 1
+		},
+		nil,
+	)
+	return err
+}
+
+func checkUnknown(s string) string {
+	if strings.Contains(s, "unknown") {
+		return "unknown"
+	}
+	return s
+}

base/metrics/metrics_logs.go

@@ -0,0 +1,49 @@
+package metrics
+
+import (
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/log"
+)
+
+func registerLogMetrics() (err error) {
+	_, err = NewFetchingCounter(
+		"logs/warning/total",
+		nil,
+		log.TotalWarningLogLines,
+		&Options{
+			Name:       "Total Warning Log Lines",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = NewFetchingCounter(
+		"logs/error/total",
+		nil,
+		log.TotalErrorLogLines,
+		&Options{
+			Name:       "Total Error Log Lines",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = NewFetchingCounter(
+		"logs/critical/total",
+		nil,
+		log.TotalCriticalLogLines,
+		&Options{
+			Name:       "Total Critical Log Lines",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

base/metrics/metrics_runtime.go

@@ -0,0 +1,98 @@
+package metrics
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+
+	vm "github.com/VictoriaMetrics/metrics"
+
+	"github.com/safing/portmaster/base/api"
+	"github.com/safing/portmaster/base/config"
+	"github.com/safing/portmaster/base/log"
+)
+
+func registerRuntimeMetric() error {
+	runtimeBase, err := newMetricBase("_runtime", nil, Options{
+		Name:           "Golang Runtime",
+		Permission:     api.PermitAdmin,
+		ExpertiseLevel: config.ExpertiseLevelDeveloper,
+	})
+	if err != nil {
+		return err
+	}
+
+	return register(&runtimeMetrics{
+		metricBase: runtimeBase,
+	})
+}
+
+type runtimeMetrics struct {
+	*metricBase
+}
+
+func (r *runtimeMetrics) WritePrometheus(w io.Writer) {
+	// If there nothing to change, just write directly to w.
+	if metricNamespace == "" && len(globalLabels) == 0 {
+		vm.WriteProcessMetrics(w)
+		return
+	}
+
+	// Write metrics to buffer.
+	buf := new(bytes.Buffer)
+	vm.WriteProcessMetrics(buf)
+
+	// Add namespace and label per line.
+	scanner := bufio.NewScanner(buf)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// Add namespace, if set.
+		if metricNamespace != "" {
+			line = metricNamespace + "_" + line
+		}
+
+		// Add global labels, if set.
+		if len(globalLabels) > 0 {
+			// Find where to insert.
+			mergeWithExisting := true
+			insertAt := strings.Index(line, "{") + 1
+			if insertAt <= 0 {
+				mergeWithExisting = false
+				insertAt = strings.Index(line, " ")
+				if insertAt < 0 {
+					continue
+				}
+			}
+
+			// Write new line directly to w.
+			fmt.Fprint(w, line[:insertAt])
+			if !mergeWithExisting {
+				fmt.Fprint(w, "{")
+			}
+			labelsAdded := 0
+			for labelKey, labelValue := range globalLabels {
+				fmt.Fprintf(w, "%s=%q", labelKey, labelValue)
+				// Add separator if not last label.
+				labelsAdded++
+				if labelsAdded < len(globalLabels) {
+					fmt.Fprint(w, ", ")
+				}
+			}
+			if mergeWithExisting {
+				fmt.Fprint(w, ", ")
+			} else {
+				fmt.Fprint(w, "}")
+			}
+			fmt.Fprintln(w, line[insertAt:])
+		}
+	}
+
+	// Check if there was an error in the scanner.
+	if scanner.Err() != nil {
+		log.Warningf("metrics: failed to scan go process metrics: %s", scanner.Err())
+	}
+}

base/metrics/module.go

@@ -0,0 +1,216 @@
+package metrics
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"sync/atomic"
+
+	"github.com/safing/portmaster/service/mgr"
+)
+
+type Metrics struct {
+	mgr      *mgr.Manager
+	instance instance
+
+	metricTicker *mgr.SleepyTicker
+}
+
+func (met *Metrics) Manager() *mgr.Manager {
+	return met.mgr
+}
+
+func (met *Metrics) Start() error {
+	return start()
+}
+
+func (met *Metrics) Stop() error {
+	return stop()
+}
+
+func (met *Metrics) SetSleep(enabled bool) {
+	if met.metricTicker != nil {
+		met.metricTicker.SetSleep(enabled)
+	}
+}
+
+var (
+	module     *Metrics
+	shimLoaded atomic.Bool
+
+	registry     []Metric
+	registryLock sync.RWMutex
+
+	readyToRegister       bool
+	firstMetricRegistered bool
+	metricNamespace       string
+	globalLabels          = make(map[string]string)
+
+	// ErrAlreadyStarted is returned when an operation is only valid before the
+	// first metric is registered, and is called after.
+	ErrAlreadyStarted = errors.New("can only be changed before first metric is registered")
+
+	// ErrAlreadyRegistered is returned when a metric with the same ID is
+	// registered again.
+	ErrAlreadyRegistered = errors.New("metric already registered")
+
+	// ErrAlreadySet is returned when a value is already set and cannot be changed.
+	ErrAlreadySet = errors.New("already set")
+
+	// ErrInvalidOptions is returned when invalid options where provided.
+	ErrInvalidOptions = errors.New("invalid options")
+)
+
+func start() error {
+	// Add metric instance name as global variable if set.
+	if instanceOption() != "" {
+		if err := AddGlobalLabel("instance", instanceOption()); err != nil {
+			return err
+		}
+	}
+
+	// Mark registry as ready to register metrics.
+	func() {
+		registryLock.Lock()
+		defer registryLock.Unlock()
+		readyToRegister = true
+	}()
+
+	if err := registerInfoMetric(); err != nil {
+		return err
+	}
+
+	if err := registerRuntimeMetric(); err != nil {
+		return err
+	}
+
+	if err := registerHostMetrics(); err != nil {
+		return err
+	}
+
+	if err := registerLogMetrics(); err != nil {
+		return err
+	}
+
+	if pushOption() != "" {
+		module.mgr.Go("metric pusher", metricsWriter)
+	}
+
+	return nil
+}
+
+func stop() error {
+	// Wait until the metrics pusher is done, as it may have started reporting
+	// and may report a higher number than we store to disk. For persistent
+	// metrics it can then happen that the first report is lower than the
+	// previous report, making prometheus think that all that happened since the
+	// last report, due to the automatic restart detection.
+
+	// The registry is read locked when writing metrics.
+	// Write lock the registry to make sure all writes are finished.
+	registryLock.Lock()
+	registryLock.Unlock() //nolint:staticcheck
+
+	storePersistentMetrics()
+
+	return nil
+}
+
+func register(m Metric) error {
+	registryLock.Lock()
+	defer registryLock.Unlock()
+
+	// Check if metric ID is already registered.
+	for _, registeredMetric := range registry {
+		if m.LabeledID() == registeredMetric.LabeledID() {
+			return ErrAlreadyRegistered
+		}
+		if m.Opts().InternalID != "" &&
+			m.Opts().InternalID == registeredMetric.Opts().InternalID {
+			return fmt.Errorf("%w with this internal ID", ErrAlreadyRegistered)
+		}
+	}
+
+	// Add new metric to registry and sort it.
+	registry = append(registry, m)
+	sort.Sort(byLabeledID(registry))
+
+	// Check if we can already register.
+	if !readyToRegister {
+		return fmt.Errorf("registering metric %q too early", m.ID())
+	}
+
+	// Set flag that first metric is now registered.
+	firstMetricRegistered = true
+
+	return nil
+}
+
+// SetNamespace sets the namespace for all metrics. It is prefixed to all
+// metric IDs.
+// It must be set before any metric is registered.
+// Does not affect golang runtime metrics.
+func SetNamespace(namespace string) error {
+	// Lock registry and check if a first metric is already registered.
+	registryLock.Lock()
+	defer registryLock.Unlock()
+	if firstMetricRegistered {
+		return ErrAlreadyStarted
+	}
+
+	// Check if the namespace is already set.
+	if metricNamespace != "" {
+		return ErrAlreadySet
+	}
+
+	metricNamespace = namespace
+	return nil
+}
+
+// AddGlobalLabel adds a global label to all metrics.
+// Global labels must be added before any metric is registered.
+// Does not affect golang runtime metrics.
+func AddGlobalLabel(name, value string) error {
+	// Lock registry and check if a first metric is already registered.
+	registryLock.Lock()
+	defer registryLock.Unlock()
+	if firstMetricRegistered {
+		return ErrAlreadyStarted
+	}
+
+	// Check format.
+	if !prometheusFormat.MatchString(name) {
+		return fmt.Errorf("metric label name %q must match %s", name, PrometheusFormatRequirement)
+	}
+
+	globalLabels[name] = value
+	return nil
+}
+
+type byLabeledID []Metric
+
+func (r byLabeledID) Len() int           { return len(r) }
+func (r byLabeledID) Less(i, j int) bool { return r[i].LabeledID() < r[j].LabeledID() }
+func (r byLabeledID) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+
+func New(instance instance) (*Metrics, error) {
+	if !shimLoaded.CompareAndSwap(false, true) {
+		return nil, errors.New("only one instance allowed")
+	}
+	m := mgr.New("Metrics")
+	module = &Metrics{
+		mgr:      m,
+		instance: instance,
+	}
+	if err := prepConfig(); err != nil {
+		return nil, err
+	}
+
+	if err := registerAPI(); err != nil {
+		return nil, err
+	}
+	return module, nil
+}
+
+type instance interface{}

base/metrics/persistence.go

@@ -0,0 +1,153 @@
+package metrics
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portmaster/base/database"
+	"github.com/safing/portmaster/base/database/record"
+	"github.com/safing/portmaster/base/log"
+)
+
+var (
+	storage       *metricsStorage
+	storageKey    string
+	storageInit   = abool.New()
+	storageLoaded = abool.New()
+
+	db = database.NewInterface(&database.Options{
+		Local:    true,
+		Internal: true,
+	})
+
+	// ErrAlreadyInitialized is returned when trying to initialize an option
+	// more than once or if the time window for initializing is over.
+	ErrAlreadyInitialized = errors.New("already initialized")
+)
+
+type metricsStorage struct {
+	sync.Mutex
+	record.Base
+
+	Start    time.Time
+	Counters map[string]uint64
+}
+
+// EnableMetricPersistence enables metric persistence for metrics that opted
+// for it. They given key is the database key where the metric data will be
+// persisted.
+// This call also directly loads the stored data from the database.
+// The returned error is only about loading the metrics, not about enabling
+// persistence.
+// May only be called once.
+func EnableMetricPersistence(key string) error {
+	// Check if already initialized.
+	if !storageInit.SetToIf(false, true) {
+		return ErrAlreadyInitialized
+	}
+
+	// Set storage key.
+	storageKey = key
+
+	// Load metrics from storage.
+	var err error
+	storage, err = getMetricsStorage(storageKey)
+	switch {
+	case err == nil:
+		// Continue.
+	case errors.Is(err, database.ErrNotFound):
+		return nil
+	default:
+		return err
+	}
+	storageLoaded.Set()
+
+	// Load saved state for all counter metrics.
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	for _, m := range registry {
+		counter, ok := m.(*Counter)
+		if ok {
+			counter.loadState()
+		}
+	}
+
+	return nil
+}
+
+func (c *Counter) loadState() {
+	// Check if we can and should load the state.
+	if !storageLoaded.IsSet() || !c.Opts().Persist {
+		return
+	}
+
+	c.Set(storage.Counters[c.LabeledID()])
+}
+
+func storePersistentMetrics() {
+	// Check if persistence is enabled.
+	if !storageInit.IsSet() || storageKey == "" {
+		return
+	}
+
+	// Create new storage.
+	newStorage := &metricsStorage{
+		// TODO: This timestamp should be taken from previous save, if possible.
+		Start:    time.Now(),
+		Counters: make(map[string]uint64),
+	}
+	newStorage.SetKey(storageKey)
+	// Copy values from previous version.
+	if storageLoaded.IsSet() {
+		newStorage.Start = storage.Start
+	}
+
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	// Export all counter metrics.
+	for _, m := range registry {
+		if m.Opts().Persist {
+			counter, ok := m.(*Counter)
+			if ok {
+				newStorage.Counters[m.LabeledID()] = counter.Get()
+			}
+		}
+	}
+
+	// Save to database.
+	err := db.Put(newStorage)
+	if err != nil {
+		log.Warningf("metrics: failed to save metrics storage to db: %s", err)
+	}
+}
+
+func getMetricsStorage(key string) (*metricsStorage, error) {
+	r, err := db.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// unwrap
+	if r.IsWrapped() {
+		// only allocate a new struct, if we need it
+		newStorage := &metricsStorage{}
+		err = record.Unwrap(r, newStorage)
+		if err != nil {
+			return nil, err
+		}
+		return newStorage, nil
+	}
+
+	// or adjust type
+	newStorage, ok := r.(*metricsStorage)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *metricsStorage, but %T", r)
+	}
+	return newStorage, nil
+}

base/metrics/testdata/.gitignore

@@ -0,0 +1 @@
+data

base/metrics/testdata/README.md

@@ -0,0 +1,4 @@
+# Testing metrics
+
+You can spin up a test setup for pushing and viewing metrics with `docker-compose up`.
+Then use the flag `--push-metrics http://127.0.0.1:8428/api/v1/import/prometheus` to push metrics.

base/metrics/testdata/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+networks:
+  pm-metrics-test-net:
+
+services:
+
+  victoriametrics:
+    container_name: pm-metrics-test-victoriametrics
+    image: victoriametrics/victoria-metrics
+    command:
+      - '--storageDataPath=/storage'
+    ports:
+      - 8428:8428
+    volumes:
+      - ./data/victoriametrics:/storage
+    networks:
+      - pm-metrics-test-net
+    restart: always
+
+  grafana:
+    container_name: pm-metrics-test-grafana
+    image: grafana/grafana
+    command:
+      - '--config=/etc/grafana/provisioning/config.ini'
+    depends_on:
+      - "victoriametrics"
+    ports:
+      - 3000:3000
+    volumes:
+      - ./data/grafana:/var/lib/grafana
+      - ./grafana:/etc/grafana/provisioning
+      - ./dashboards:/dashboards
+    networks:
+      - pm-metrics-test-net
+    restart: always

base/metrics/testdata/grafana/config.ini

@@ -0,0 +1,10 @@
+[auth]
+disable_login_form = true
+disable_signout_menu = true
+
+[auth.basic]
+enabled = false
+
+[auth.anonymous]
+enabled = true
+org_role = Admin

base/metrics/testdata/grafana/dashboards/portmaster.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'Portmaster'
+    folder: 'Portmaster'
+    disableDeletion: true
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /dashboards
+      foldersFromFilesStructure: true

base/metrics/testdata/grafana/datasources/datasource.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+    - name: VictoriaMetrics
+      type: prometheus
+      access: proxy
+      url: http://pm-metrics-test-victoriametrics:8428
+      isDefault: true