Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

base/database/record/base.go

@@ -0,0 +1,156 @@
+package record
+
+import (
+	"errors"
+
+	"github.com/safing/portmaster/base/database/accessor"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
+)
+
+// TODO(ppacher):
+//		we can reduce the record.Record interface a lot by moving
+//		most of those functions that require the Record as it's first
+//		parameter to static package functions
+//		(i.e. Marshal, MarshalRecord, GetAccessor, ...).
+//		We should also consider given Base a GetBase() *Base method
+//		that returns itself. This way we can remove almost all Base
+//		only methods from the record.Record interface. That is, we can
+//		remove all those CreateMeta, UpdateMeta, ... stuff from the
+//		interface definition (not the actual functions!). This would make
+// 		the record.Record interface slim and only provide methods that
+//		most users actually need. All those database/storage related methods
+// 		can still be accessed by using GetBase().XXX() instead. We can also
+//		expose the dbName and dbKey and meta properties directly which would
+// 		make a nice JSON blob when marshalled.
+
+// Base provides a quick way to comply with the Model interface.
+type Base struct {
+	dbName string
+	dbKey  string
+	meta   *Meta
+}
+
+// SetKey sets the key on the database record. The key may only be set once and
+// future calls to SetKey will be ignored. If you want to copy/move the record
+// to another database key, you will need to create a copy and assign a new key.
+// A key must be set before the record is used in any database operation.
+func (b *Base) SetKey(key string) {
+	if !b.KeyIsSet() {
+		b.dbName, b.dbKey = ParseKey(key)
+	} else {
+		log.Errorf("database: key is already set: tried to replace %q with %q", b.Key(), key)
+	}
+}
+
+// ResetKey resets the database name and key.
+// Use with caution!
+func (b *Base) ResetKey() {
+	b.dbName = ""
+	b.dbKey = ""
+}
+
+// Key returns the key of the database record.
+// As the key must be set before any usage and can only be set once, this
+// function may be used without locking the record.
+func (b *Base) Key() string {
+	return b.dbName + ":" + b.dbKey
+}
+
+// KeyIsSet returns true if the database key is set.
+// As the key must be set before any usage and can only be set once, this
+// function may be used without locking the record.
+func (b *Base) KeyIsSet() bool {
+	return b.dbName != ""
+}
+
+// DatabaseName returns the name of the database.
+// As the key must be set before any usage and can only be set once, this
+// function may be used without locking the record.
+func (b *Base) DatabaseName() string {
+	return b.dbName
+}
+
+// DatabaseKey returns the database key of the database record.
+// As the key must be set before any usage and can only be set once, this
+// function may be used without locking the record.
+func (b *Base) DatabaseKey() string {
+	return b.dbKey
+}
+
+// Meta returns the metadata object for this record.
+func (b *Base) Meta() *Meta {
+	return b.meta
+}
+
+// CreateMeta sets a default metadata object for this record.
+func (b *Base) CreateMeta() {
+	b.meta = &Meta{}
+}
+
+// UpdateMeta creates the metadata if it does not exist and updates it.
+func (b *Base) UpdateMeta() {
+	if b.meta == nil {
+		b.CreateMeta()
+	}
+	b.meta.Update()
+}
+
+// SetMeta sets the metadata on the database record, it should only be called after loading the record. Use MoveTo to save the record with another key.
+func (b *Base) SetMeta(meta *Meta) {
+	b.meta = meta
+}
+
+// Marshal marshals the object, without the database key or metadata. It returns nil if the record is deleted.
+func (b *Base) Marshal(self Record, format uint8) ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if b.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
+	dumped, err := dsd.Dump(self, format)
+	if err != nil {
+		return nil, err
+	}
+	return dumped, nil
+}
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (b *Base) MarshalRecord(self Record) ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta encoding
+	metaSection, err := dsd.Dump(b.meta, dsd.GenCode)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := b.Marshal(self, dsd.JSON)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}
+
+// IsWrapped returns whether the record is a Wrapper.
+func (b *Base) IsWrapped() bool {
+	return false
+}
+
+// GetAccessor returns an accessor for this record, if available.
+func (b *Base) GetAccessor(self Record) accessor.Accessor {
+	return accessor.NewStructAccessor(self)
+}

base/database/record/base_test.go

@@ -0,0 +1,13 @@
+package record
+
+import "testing"
+
+func TestBaseRecord(t *testing.T) {
+	t.Parallel()
+
+	// check model interface compliance
+	var m Record
+	b := &TestRecord{}
+	m = b
+	_ = m
+}

base/database/record/key.go

@@ -0,0 +1,14 @@
+package record
+
+import (
+	"strings"
+)
+
+// ParseKey splits a key into it's database name and key parts.
+func ParseKey(key string) (dbName, dbKey string) {
+	splitted := strings.SplitN(key, ":", 2)
+	if len(splitted) < 2 {
+		return splitted[0], ""
+	}
+	return splitted[0], strings.Join(splitted[1:], ":")
+}

base/database/record/meta-bench_test.go

@@ -0,0 +1,348 @@
+package record
+
+// Benchmark:
+// BenchmarkAllocateBytes-8                	2000000000	         0.76 ns/op
+// BenchmarkAllocateStruct1-8              	2000000000	         0.76 ns/op
+// BenchmarkAllocateStruct2-8              	2000000000	         0.79 ns/op
+// BenchmarkMetaSerializeContainer-8       	 1000000	      1703 ns/op
+// BenchmarkMetaUnserializeContainer-8     	 2000000	       950 ns/op
+// BenchmarkMetaSerializeVarInt-8          	 3000000	       457 ns/op
+// BenchmarkMetaUnserializeVarInt-8        	20000000	        62.9 ns/op
+// BenchmarkMetaSerializeWithXDR2-8        	 1000000	      2360 ns/op
+// BenchmarkMetaUnserializeWithXDR2-8      	  500000	      3189 ns/op
+// BenchmarkMetaSerializeWithColfer-8      	10000000	       237 ns/op
+// BenchmarkMetaUnserializeWithColfer-8    	20000000	        51.7 ns/op
+// BenchmarkMetaSerializeWithCodegen-8     	50000000	        23.7 ns/op
+// BenchmarkMetaUnserializeWithCodegen-8   	100000000	        18.9 ns/op
+// BenchmarkMetaSerializeWithDSDJSON-8     	 1000000	      2398 ns/op
+// BenchmarkMetaUnserializeWithDSDJSON-8   	  300000	      6264 ns/op
+
+import (
+	"testing"
+	"time"
+
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
+	"github.com/safing/structures/varint"
+)
+
+var testMeta = &Meta{
+	Created:   time.Now().Unix(),
+	Modified:  time.Now().Unix(),
+	Expires:   time.Now().Unix(),
+	Deleted:   time.Now().Unix(),
+	secret:    true,
+	cronjewel: true,
+}
+
+func BenchmarkAllocateBytes(b *testing.B) {
+	for range b.N {
+		_ = make([]byte, 33)
+	}
+}
+
+func BenchmarkAllocateStruct1(b *testing.B) {
+	for range b.N {
+		var newMeta Meta
+		_ = newMeta
+	}
+}
+
+func BenchmarkAllocateStruct2(b *testing.B) {
+	for range b.N {
+		_ = Meta{}
+	}
+}
+
+func BenchmarkMetaSerializeContainer(b *testing.B) {
+	// Start benchmark
+	for range b.N {
+		c := container.New()
+		c.AppendNumber(uint64(testMeta.Created))
+		c.AppendNumber(uint64(testMeta.Modified))
+		c.AppendNumber(uint64(testMeta.Expires))
+		c.AppendNumber(uint64(testMeta.Deleted))
+		switch {
+		case testMeta.secret && testMeta.cronjewel:
+			c.AppendNumber(3)
+		case testMeta.secret:
+			c.AppendNumber(1)
+		case testMeta.cronjewel:
+			c.AppendNumber(2)
+		default:
+			c.AppendNumber(0)
+		}
+	}
+}
+
+func BenchmarkMetaUnserializeContainer(b *testing.B) {
+	// Setup
+	c := container.New()
+	c.AppendNumber(uint64(testMeta.Created))
+	c.AppendNumber(uint64(testMeta.Modified))
+	c.AppendNumber(uint64(testMeta.Expires))
+	c.AppendNumber(uint64(testMeta.Deleted))
+	switch {
+	case testMeta.secret && testMeta.cronjewel:
+		c.AppendNumber(3)
+	case testMeta.secret:
+		c.AppendNumber(1)
+	case testMeta.cronjewel:
+		c.AppendNumber(2)
+	default:
+		c.AppendNumber(0)
+	}
+	encodedData := c.CompileData()
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for range b.N {
+		var newMeta Meta
+		var err error
+		var num uint64
+		c := container.New(encodedData)
+		num, err = c.GetNextN64()
+		newMeta.Created = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Modified = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Expires = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+		num, err = c.GetNextN64()
+		newMeta.Deleted = int64(num)
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		flags, err := c.GetNextN8()
+		if err != nil {
+			b.Errorf("could not decode: %s", err)
+			return
+		}
+
+		switch flags {
+		case 3:
+			newMeta.secret = true
+			newMeta.cronjewel = true
+		case 2:
+			newMeta.cronjewel = true
+		case 1:
+			newMeta.secret = true
+		case 0:
+		default:
+			b.Errorf("invalid flag value: %d", flags)
+			return
+		}
+	}
+}
+
+func BenchmarkMetaSerializeVarInt(b *testing.B) {
+	// Start benchmark
+	for range b.N {
+		encoded := make([]byte, 33)
+		offset := 0
+		data := varint.Pack64(uint64(testMeta.Created))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.Modified))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.Expires))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+		data = varint.Pack64(uint64(testMeta.Deleted))
+		for _, part := range data {
+			encoded[offset] = part
+			offset++
+		}
+
+		switch {
+		case testMeta.secret && testMeta.cronjewel:
+			encoded[offset] = 3
+		case testMeta.secret:
+			encoded[offset] = 1
+		case testMeta.cronjewel:
+			encoded[offset] = 2
+		default:
+			encoded[offset] = 0
+		}
+	}
+}
+
+func BenchmarkMetaUnserializeVarInt(b *testing.B) {
+	// Setup
+	encoded := make([]byte, 33)
+	offset := 0
+	data := varint.Pack64(uint64(testMeta.Created))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.Modified))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.Expires))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+	data = varint.Pack64(uint64(testMeta.Deleted))
+	for _, part := range data {
+		encoded[offset] = part
+		offset++
+	}
+
+	switch {
+	case testMeta.secret && testMeta.cronjewel:
+		encoded[offset] = 3
+	case testMeta.secret:
+		encoded[offset] = 1
+	case testMeta.cronjewel:
+		encoded[offset] = 2
+	default:
+		encoded[offset] = 0
+	}
+	offset++
+	encodedData := encoded[:offset]
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for range b.N {
+		var newMeta Meta
+		offset = 0
+
+		num, n, err := varint.Unpack64(encodedData)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.Created = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.Modified = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.Expires = int64(num)
+		offset += n
+
+		num, n, err = varint.Unpack64(encodedData[offset:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		testMeta.Deleted = int64(num)
+		offset += n
+
+		switch encodedData[offset] {
+		case 3:
+			newMeta.secret = true
+			newMeta.cronjewel = true
+		case 2:
+			newMeta.cronjewel = true
+		case 1:
+			newMeta.secret = true
+		case 0:
+		default:
+			b.Errorf("invalid flag value: %d", encodedData[offset])
+			return
+		}
+	}
+}
+
+func BenchmarkMetaSerializeWithCodegen(b *testing.B) {
+	for range b.N {
+		_, err := testMeta.GenCodeMarshal(nil)
+		if err != nil {
+			b.Errorf("failed to serialize with codegen: %s", err)
+			return
+		}
+	}
+}
+
+func BenchmarkMetaUnserializeWithCodegen(b *testing.B) {
+	// Setup
+	encodedData, err := testMeta.GenCodeMarshal(nil)
+	if err != nil {
+		b.Errorf("failed to serialize with codegen: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for range b.N {
+		var newMeta Meta
+		_, err := newMeta.GenCodeUnmarshal(encodedData)
+		if err != nil {
+			b.Errorf("failed to unserialize with codegen: %s", err)
+			return
+		}
+	}
+}
+
+func BenchmarkMetaSerializeWithDSDJSON(b *testing.B) {
+	for range b.N {
+		_, err := dsd.Dump(testMeta, dsd.JSON)
+		if err != nil {
+			b.Errorf("failed to serialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+}
+
+func BenchmarkMetaUnserializeWithDSDJSON(b *testing.B) {
+	// Setup
+	encodedData, err := dsd.Dump(testMeta, dsd.JSON)
+	if err != nil {
+		b.Errorf("failed to serialize with DSD/JSON: %s", err)
+		return
+	}
+
+	// Reset timer for precise results
+	b.ResetTimer()
+
+	// Start benchmark
+	for range b.N {
+		var newMeta Meta
+		_, err := dsd.Load(encodedData, &newMeta)
+		if err != nil {
+			b.Errorf("failed to unserialize with DSD/JSON: %s", err)
+			return
+		}
+	}
+}

base/database/record/meta-gencode.go

@@ -0,0 +1,145 @@
+package record
+
+import (
+	"fmt"
+)
+
+// GenCodeSize returns the size of the gencode marshalled byte slice.
+func (m *Meta) GenCodeSize() (s int) {
+	s += 34
+	return
+}
+
+// GenCodeMarshal gencode marshalls Meta into the given byte array, or a new one if its too small.
+func (m *Meta) GenCodeMarshal(buf []byte) ([]byte, error) {
+	size := m.GenCodeSize()
+	{
+		if cap(buf) >= size {
+			buf = buf[:size]
+		} else {
+			buf = make([]byte, size)
+		}
+	}
+	i := uint64(0)
+
+	{
+
+		buf[0+0] = byte(m.Created >> 0)
+
+		buf[1+0] = byte(m.Created >> 8)
+
+		buf[2+0] = byte(m.Created >> 16)
+
+		buf[3+0] = byte(m.Created >> 24)
+
+		buf[4+0] = byte(m.Created >> 32)
+
+		buf[5+0] = byte(m.Created >> 40)
+
+		buf[6+0] = byte(m.Created >> 48)
+
+		buf[7+0] = byte(m.Created >> 56)
+
+	}
+	{
+
+		buf[0+8] = byte(m.Modified >> 0)
+
+		buf[1+8] = byte(m.Modified >> 8)
+
+		buf[2+8] = byte(m.Modified >> 16)
+
+		buf[3+8] = byte(m.Modified >> 24)
+
+		buf[4+8] = byte(m.Modified >> 32)
+
+		buf[5+8] = byte(m.Modified >> 40)
+
+		buf[6+8] = byte(m.Modified >> 48)
+
+		buf[7+8] = byte(m.Modified >> 56)
+
+	}
+	{
+
+		buf[0+16] = byte(m.Expires >> 0)
+
+		buf[1+16] = byte(m.Expires >> 8)
+
+		buf[2+16] = byte(m.Expires >> 16)
+
+		buf[3+16] = byte(m.Expires >> 24)
+
+		buf[4+16] = byte(m.Expires >> 32)
+
+		buf[5+16] = byte(m.Expires >> 40)
+
+		buf[6+16] = byte(m.Expires >> 48)
+
+		buf[7+16] = byte(m.Expires >> 56)
+
+	}
+	{
+
+		buf[0+24] = byte(m.Deleted >> 0)
+
+		buf[1+24] = byte(m.Deleted >> 8)
+
+		buf[2+24] = byte(m.Deleted >> 16)
+
+		buf[3+24] = byte(m.Deleted >> 24)
+
+		buf[4+24] = byte(m.Deleted >> 32)
+
+		buf[5+24] = byte(m.Deleted >> 40)
+
+		buf[6+24] = byte(m.Deleted >> 48)
+
+		buf[7+24] = byte(m.Deleted >> 56)
+
+	}
+	{
+		if m.secret {
+			buf[32] = 1
+		} else {
+			buf[32] = 0
+		}
+	}
+	{
+		if m.cronjewel {
+			buf[33] = 1
+		} else {
+			buf[33] = 0
+		}
+	}
+	return buf[:i+34], nil
+}
+
+// GenCodeUnmarshal gencode unmarshalls Meta and returns the bytes read.
+func (m *Meta) GenCodeUnmarshal(buf []byte) (uint64, error) {
+	if len(buf) < m.GenCodeSize() {
+		return 0, fmt.Errorf("insufficient data: got %d out of %d bytes", len(buf), m.GenCodeSize())
+	}
+
+	i := uint64(0)
+
+	{
+		m.Created = 0 | (int64(buf[0+0]) << 0) | (int64(buf[1+0]) << 8) | (int64(buf[2+0]) << 16) | (int64(buf[3+0]) << 24) | (int64(buf[4+0]) << 32) | (int64(buf[5+0]) << 40) | (int64(buf[6+0]) << 48) | (int64(buf[7+0]) << 56)
+	}
+	{
+		m.Modified = 0 | (int64(buf[0+8]) << 0) | (int64(buf[1+8]) << 8) | (int64(buf[2+8]) << 16) | (int64(buf[3+8]) << 24) | (int64(buf[4+8]) << 32) | (int64(buf[5+8]) << 40) | (int64(buf[6+8]) << 48) | (int64(buf[7+8]) << 56)
+	}
+	{
+		m.Expires = 0 | (int64(buf[0+16]) << 0) | (int64(buf[1+16]) << 8) | (int64(buf[2+16]) << 16) | (int64(buf[3+16]) << 24) | (int64(buf[4+16]) << 32) | (int64(buf[5+16]) << 40) | (int64(buf[6+16]) << 48) | (int64(buf[7+16]) << 56)
+	}
+	{
+		m.Deleted = 0 | (int64(buf[0+24]) << 0) | (int64(buf[1+24]) << 8) | (int64(buf[2+24]) << 16) | (int64(buf[3+24]) << 24) | (int64(buf[4+24]) << 32) | (int64(buf[5+24]) << 40) | (int64(buf[6+24]) << 48) | (int64(buf[7+24]) << 56)
+	}
+	{
+		m.secret = buf[32] == 1
+	}
+	{
+		m.cronjewel = buf[33] == 1
+	}
+	return i + 34, nil
+}

base/database/record/meta-gencode_test.go

@@ -0,0 +1,35 @@
+package record
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+var genCodeTestMeta = &Meta{
+	Created:   time.Now().Unix(),
+	Modified:  time.Now().Unix(),
+	Expires:   time.Now().Unix(),
+	Deleted:   time.Now().Unix(),
+	secret:    true,
+	cronjewel: true,
+}
+
+func TestGenCode(t *testing.T) {
+	t.Parallel()
+
+	encoded, err := genCodeTestMeta.GenCodeMarshal(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newMeta := &Meta{}
+	_, err = newMeta.GenCodeUnmarshal(encoded)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(genCodeTestMeta, newMeta) {
+		t.Errorf("objects are not equal, got: %v", newMeta)
+	}
+}

base/database/record/meta.colf

@@ -0,0 +1,10 @@
+package record
+
+type course struct {
+  Created   int64
+  Modified  int64
+  Expires   int64
+  Deleted   int64
+  Secret    bool
+  Cronjewel bool
+}

base/database/record/meta.gencode

@@ -0,0 +1,8 @@
+struct Meta {
+	Created   int64
+	Modified  int64
+	Expires   int64
+	Deleted   int64
+	Secret    bool
+	Cronjewel bool
+}

base/database/record/meta.go

@@ -0,0 +1,129 @@
+package record
+
+import "time"
+
+// Meta holds metadata about the record.
+type Meta struct {
+	Created   int64
+	Modified  int64
+	Expires   int64
+	Deleted   int64
+	secret    bool // secrets must not be sent to the UI, only synced between nodes
+	cronjewel bool // crownjewels must never leave the instance, but may be read by the UI
+}
+
+// SetAbsoluteExpiry sets an absolute expiry time (in seconds), that is not affected when the record is updated.
+func (m *Meta) SetAbsoluteExpiry(seconds int64) {
+	m.Expires = seconds
+	m.Deleted = 0
+}
+
+// SetRelativateExpiry sets a relative expiry time (ie. TTL in seconds) that is automatically updated whenever the record is updated/saved.
+func (m *Meta) SetRelativateExpiry(seconds int64) {
+	if seconds >= 0 {
+		m.Deleted = -seconds
+	}
+}
+
+// GetAbsoluteExpiry returns the absolute expiry time.
+func (m *Meta) GetAbsoluteExpiry() int64 {
+	return m.Expires
+}
+
+// GetRelativeExpiry returns the current relative expiry time - ie. seconds until expiry.
+// A negative value signifies that the record does not expire.
+func (m *Meta) GetRelativeExpiry() int64 {
+	if m.Expires == 0 {
+		return -1
+	}
+
+	abs := m.Expires - time.Now().Unix()
+	if abs < 0 {
+		return 0
+	}
+	return abs
+}
+
+// MakeCrownJewel marks the database records as a crownjewel, meaning that it will not be sent/synced to other devices.
+func (m *Meta) MakeCrownJewel() {
+	m.cronjewel = true
+}
+
+// MakeSecret sets the database record as secret, meaning that it may only be used internally, and not by interfacing processes, such as the UI.
+func (m *Meta) MakeSecret() {
+	m.secret = true
+}
+
+// Update updates the internal meta states and should be called before writing the record to the database.
+func (m *Meta) Update() {
+	now := time.Now().Unix()
+	m.Modified = now
+	if m.Created == 0 {
+		m.Created = now
+	}
+	if m.Deleted < 0 {
+		m.Expires = now - m.Deleted
+	}
+}
+
+// Reset resets all metadata, except for the secret and crownjewel status.
+func (m *Meta) Reset() {
+	m.Created = 0
+	m.Modified = 0
+	m.Expires = 0
+	m.Deleted = 0
+}
+
+// Delete marks the record as deleted.
+func (m *Meta) Delete() {
+	m.Deleted = time.Now().Unix()
+}
+
+// IsDeleted returns whether the record is deleted.
+func (m *Meta) IsDeleted() bool {
+	return m.Deleted > 0
+}
+
+// CheckValidity checks whether the database record is valid.
+func (m *Meta) CheckValidity() (valid bool) {
+	if m == nil {
+		return false
+	}
+
+	switch {
+	case m.Deleted > 0:
+		return false
+	case m.Expires > 0 && m.Expires < time.Now().Unix():
+		return false
+	default:
+		return true
+	}
+}
+
+// CheckPermission checks whether the database record may be accessed with the following scope.
+func (m *Meta) CheckPermission(local, internal bool) (permitted bool) {
+	if m == nil {
+		return false
+	}
+
+	switch {
+	case !local && m.cronjewel:
+		return false
+	case !internal && m.secret:
+		return false
+	default:
+		return true
+	}
+}
+
+// Duplicate returns a new copy of Meta.
+func (m *Meta) Duplicate() *Meta {
+	return &Meta{
+		Created:   m.Created,
+		Modified:  m.Modified,
+		Expires:   m.Expires,
+		Deleted:   m.Deleted,
+		secret:    m.secret,
+		cronjewel: m.cronjewel,
+	}
+}

base/database/record/record.go

@@ -0,0 +1,32 @@
+package record
+
+import (
+	"github.com/safing/portmaster/base/database/accessor"
+)
+
+// Record provides an interface for uniformally handling database records.
+type Record interface {
+	SetKey(key string) // test:config
+	Key() string       // test:config
+	KeyIsSet() bool
+	DatabaseName() string // test
+	DatabaseKey() string  // config
+
+	// Metadata.
+	Meta() *Meta
+	SetMeta(meta *Meta)
+	CreateMeta()
+	UpdateMeta()
+
+	// Serialization.
+	Marshal(self Record, format uint8) ([]byte, error)
+	MarshalRecord(self Record) ([]byte, error)
+	GetAccessor(self Record) accessor.Accessor
+
+	// Locking.
+	Lock()
+	Unlock()
+
+	// Wrapping.
+	IsWrapped() bool
+}

base/database/record/record_test.go

@@ -0,0 +1,10 @@
+package record
+
+import (
+	"sync"
+)
+
+type TestRecord struct {
+	Base
+	sync.Mutex
+}

base/database/record/wrapper.go

@@ -0,0 +1,160 @@
+package record
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/safing/portmaster/base/database/accessor"
+	"github.com/safing/structures/container"
+	"github.com/safing/structures/dsd"
+	"github.com/safing/structures/varint"
+)
+
+// Wrapper wraps raw data and implements the Record interface.
+type Wrapper struct {
+	Base
+	sync.Mutex
+
+	Format uint8
+	Data   []byte
+}
+
+// NewRawWrapper returns a record wrapper for the given data, including metadata. This is normally only used by storage backends when loading records.
+func NewRawWrapper(database, key string, data []byte) (*Wrapper, error) {
+	version, offset, err := varint.Unpack8(data)
+	if err != nil {
+		return nil, err
+	}
+	if version != 1 {
+		return nil, fmt.Errorf("incompatible record version: %d", version)
+	}
+
+	metaSection, n, err := varint.GetNextBlock(data[offset:])
+	if err != nil {
+		return nil, fmt.Errorf("could not get meta section: %w", err)
+	}
+	offset += n
+
+	newMeta := &Meta{}
+	_, err = dsd.Load(metaSection, newMeta)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshal meta section: %w", err)
+	}
+
+	var format uint8 = dsd.RAW
+	if !newMeta.IsDeleted() {
+		format, n, err = varint.Unpack8(data[offset:])
+		if err != nil {
+			return nil, fmt.Errorf("could not get dsd format: %w", err)
+		}
+		offset += n
+	}
+
+	return &Wrapper{
+		Base{
+			database,
+			key,
+			newMeta,
+		},
+		sync.Mutex{},
+		format,
+		data[offset:],
+	}, nil
+}
+
+// NewWrapper returns a new record wrapper for the given data.
+func NewWrapper(key string, meta *Meta, format uint8, data []byte) (*Wrapper, error) {
+	dbName, dbKey := ParseKey(key)
+
+	return &Wrapper{
+		Base{
+			dbName: dbName,
+			dbKey:  dbKey,
+			meta:   meta,
+		},
+		sync.Mutex{},
+		format,
+		data,
+	}, nil
+}
+
+// Marshal marshals the object, without the database key or metadata.
+func (w *Wrapper) Marshal(r Record, format uint8) ([]byte, error) {
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if w.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
+	if format != dsd.AUTO && format != w.Format {
+		return nil, errors.New("could not dump model, wrapped object format mismatch")
+	}
+
+	data := make([]byte, len(w.Data)+1)
+	data[0] = w.Format
+	copy(data[1:], w.Data)
+
+	return data, nil
+}
+
+// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
+	// Duplication necessary, as the version from Base would call Base.Marshal instead of Wrapper.Marshal
+
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	// version
+	c := container.New([]byte{1})
+
+	// meta
+	metaSection, err := dsd.Dump(w.meta, dsd.GenCode)
+	if err != nil {
+		return nil, err
+	}
+	c.AppendAsBlock(metaSection)
+
+	// data
+	dataSection, err := w.Marshal(r, dsd.AUTO)
+	if err != nil {
+		return nil, err
+	}
+	c.Append(dataSection)
+
+	return c.CompileData(), nil
+}
+
+// IsWrapped returns whether the record is a Wrapper.
+func (w *Wrapper) IsWrapped() bool {
+	return true
+}
+
+// Unwrap unwraps data into a record.
+func Unwrap(wrapped, r Record) error {
+	wrapper, ok := wrapped.(*Wrapper)
+	if !ok {
+		return fmt.Errorf("cannot unwrap %T", wrapped)
+	}
+
+	err := dsd.LoadAsFormat(wrapper.Data, wrapper.Format, r)
+	if err != nil {
+		return fmt.Errorf("failed to unwrap %T: %w", r, err)
+	}
+
+	r.SetKey(wrapped.Key())
+	r.SetMeta(wrapped.Meta())
+
+	return nil
+}
+
+// GetAccessor returns an accessor for this record, if available.
+func (w *Wrapper) GetAccessor(self Record) accessor.Accessor {
+	if w.Format == dsd.JSON && len(w.Data) > 0 {
+		return accessor.NewJSONBytesAccessor(&w.Data)
+	}
+	return nil
+}

base/database/record/wrapper_test.go

@@ -0,0 +1,57 @@
+package record
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/safing/structures/dsd"
+)
+
+func TestWrapper(t *testing.T) {
+	t.Parallel()
+
+	// check model interface compliance
+	var m Record
+	w := &Wrapper{}
+	m = w
+	_ = m
+
+	// create test data
+	testData := []byte(`{"a": "b"}`)
+	encodedTestData := []byte(`J{"a": "b"}`)
+
+	// test wrapper
+	wrapper, err := NewWrapper("test:a", &Meta{}, dsd.JSON, testData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wrapper.Format != dsd.JSON {
+		t.Error("format mismatch")
+	}
+	if !bytes.Equal(testData, wrapper.Data) {
+		t.Error("data mismatch")
+	}
+
+	encoded, err := wrapper.Marshal(wrapper, dsd.JSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(encodedTestData, encoded) {
+		t.Error("marshal mismatch")
+	}
+
+	wrapper.SetMeta(&Meta{})
+	wrapper.meta.Update()
+	raw, err := wrapper.MarshalRecord(wrapper)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrapper2, err := NewRawWrapper("test", "a", raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(testData, wrapper2.Data) {
+		t.Error("marshal mismatch")
+	}
+}