Restructure modules (#1572)
* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
231
base/database/storage/badger/badger.go
Normal file
231
base/database/storage/badger/badger.go
Normal file
@@ -0,0 +1,231 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/badger"
|
||||
|
||||
"github.com/safing/portmaster/base/database/iterator"
|
||||
"github.com/safing/portmaster/base/database/query"
|
||||
"github.com/safing/portmaster/base/database/record"
|
||||
"github.com/safing/portmaster/base/database/storage"
|
||||
"github.com/safing/portmaster/base/log"
|
||||
)
|
||||
|
||||
// Badger database made pluggable for portbase.
|
||||
type Badger struct {
|
||||
name string
|
||||
db *badger.DB
|
||||
}
|
||||
|
||||
func init() {
|
||||
_ = storage.Register("badger", NewBadger)
|
||||
}
|
||||
|
||||
// NewBadger opens/creates a badger database.
|
||||
func NewBadger(name, location string) (storage.Interface, error) {
|
||||
opts := badger.DefaultOptions(location)
|
||||
|
||||
db, err := badger.Open(opts)
|
||||
if errors.Is(err, badger.ErrTruncateNeeded) {
|
||||
// clean up after crash
|
||||
log.Warningf("database/storage: truncating corrupted value log of badger database %s: this may cause data loss", name)
|
||||
opts.Truncate = true
|
||||
db, err = badger.Open(opts)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Badger{
|
||||
name: name,
|
||||
db: db,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get returns a database record.
|
||||
func (b *Badger) Get(key string) (record.Record, error) {
|
||||
var item *badger.Item
|
||||
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
var err error
|
||||
item, err = txn.Get([]byte(key))
|
||||
if err != nil {
|
||||
if errors.Is(err, badger.ErrKeyNotFound) {
|
||||
return storage.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// return err if deleted or expired
|
||||
if item.IsDeletedOrExpired() {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
|
||||
data, err := item.ValueCopy(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m, err := record.NewRawWrapper(b.name, string(item.Key()), data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// GetMeta returns the metadata of a database record.
|
||||
func (b *Badger) GetMeta(key string) (*record.Meta, error) {
|
||||
// TODO: Replace with more performant variant.
|
||||
|
||||
r, err := b.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return r.Meta(), nil
|
||||
}
|
||||
|
||||
// Put stores a record in the database.
|
||||
func (b *Badger) Put(r record.Record) (record.Record, error) {
|
||||
data, err := r.MarshalRecord(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = b.db.Update(func(txn *badger.Txn) error {
|
||||
return txn.Set([]byte(r.DatabaseKey()), data)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Delete deletes a record from the database.
|
||||
func (b *Badger) Delete(key string) error {
|
||||
return b.db.Update(func(txn *badger.Txn) error {
|
||||
err := txn.Delete([]byte(key))
|
||||
if err != nil && !errors.Is(err, badger.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Query returns a an iterator for the supplied query.
|
||||
func (b *Badger) Query(q *query.Query, local, internal bool) (*iterator.Iterator, error) {
|
||||
_, err := q.Check()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid query: %w", err)
|
||||
}
|
||||
|
||||
queryIter := iterator.New()
|
||||
|
||||
go b.queryExecutor(queryIter, q, local, internal)
|
||||
return queryIter, nil
|
||||
}
|
||||
|
||||
//nolint:gocognit
|
||||
func (b *Badger) queryExecutor(queryIter *iterator.Iterator, q *query.Query, local, internal bool) {
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
it := txn.NewIterator(badger.DefaultIteratorOptions)
|
||||
defer it.Close()
|
||||
prefix := []byte(q.DatabaseKeyPrefix())
|
||||
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
|
||||
item := it.Item()
|
||||
|
||||
var data []byte
|
||||
err := item.Value(func(val []byte) error {
|
||||
data = val
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := record.NewRawWrapper(b.name, string(item.Key()), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !r.Meta().CheckValidity() {
|
||||
continue
|
||||
}
|
||||
if !r.Meta().CheckPermission(local, internal) {
|
||||
continue
|
||||
}
|
||||
|
||||
if q.MatchesRecord(r) {
|
||||
copiedData, err := item.ValueCopy(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newWrapper, err := record.NewRawWrapper(b.name, r.DatabaseKey(), copiedData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-queryIter.Done:
|
||||
return nil
|
||||
case queryIter.Next <- newWrapper:
|
||||
default:
|
||||
select {
|
||||
case queryIter.Next <- newWrapper:
|
||||
case <-queryIter.Done:
|
||||
return nil
|
||||
case <-time.After(1 * time.Minute):
|
||||
return errors.New("query timeout")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
queryIter.Finish(err)
|
||||
}
|
||||
|
||||
// ReadOnly returns whether the database is read only.
|
||||
func (b *Badger) ReadOnly() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Injected returns whether the database is injected.
|
||||
func (b *Badger) Injected() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Maintain runs a light maintenance operation on the database.
|
||||
func (b *Badger) Maintain(_ context.Context) error {
|
||||
_ = b.db.RunValueLogGC(0.7)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaintainThorough runs a thorough maintenance operation on the database.
|
||||
func (b *Badger) MaintainThorough(_ context.Context) (err error) {
|
||||
for err == nil {
|
||||
err = b.db.RunValueLogGC(0.7)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaintainRecordStates maintains records states in the database.
|
||||
func (b *Badger) MaintainRecordStates(ctx context.Context, purgeDeletedBefore time.Time, shadowDelete bool) error {
|
||||
// TODO: implement MaintainRecordStates
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the database.
|
||||
func (b *Badger) Shutdown() error {
|
||||
return b.db.Close()
|
||||
}
|
||||
148
base/database/storage/badger/badger_test.go
Normal file
148
base/database/storage/badger/badger_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/safing/portmaster/base/database/query"
|
||||
"github.com/safing/portmaster/base/database/record"
|
||||
"github.com/safing/portmaster/base/database/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
// Compile time interface checks.
|
||||
_ storage.Interface = &Badger{}
|
||||
_ storage.Maintainer = &Badger{}
|
||||
)
|
||||
|
||||
type TestRecord struct { //nolint:maligned
|
||||
record.Base
|
||||
sync.Mutex
|
||||
S string
|
||||
I int
|
||||
I8 int8
|
||||
I16 int16
|
||||
I32 int32
|
||||
I64 int64
|
||||
UI uint
|
||||
UI8 uint8
|
||||
UI16 uint16
|
||||
UI32 uint32
|
||||
UI64 uint64
|
||||
F32 float32
|
||||
F64 float64
|
||||
B bool
|
||||
}
|
||||
|
||||
func TestBadger(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
testDir, err := os.MkdirTemp("", "testing-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = os.RemoveAll(testDir) // clean up
|
||||
}()
|
||||
|
||||
// start
|
||||
db, err := NewBadger("test", testDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := &TestRecord{
|
||||
S: "banana",
|
||||
I: 42,
|
||||
I8: 42,
|
||||
I16: 42,
|
||||
I32: 42,
|
||||
I64: 42,
|
||||
UI: 42,
|
||||
UI8: 42,
|
||||
UI16: 42,
|
||||
UI32: 42,
|
||||
UI64: 42,
|
||||
F32: 42.42,
|
||||
F64: 42.42,
|
||||
B: true,
|
||||
}
|
||||
a.SetMeta(&record.Meta{})
|
||||
a.Meta().Update()
|
||||
a.SetKey("test:A")
|
||||
|
||||
// put record
|
||||
_, err = db.Put(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// get and compare
|
||||
r1, err := db.Get("A")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a1 := &TestRecord{}
|
||||
err = record.Unwrap(r1, a1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(a, a1) {
|
||||
t.Fatalf("mismatch, got %v", a1)
|
||||
}
|
||||
|
||||
// test query
|
||||
q := query.New("").MustBeValid()
|
||||
it, err := db.Query(q, true, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cnt := 0
|
||||
for range it.Next {
|
||||
cnt++
|
||||
}
|
||||
if it.Err() != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cnt != 1 {
|
||||
t.Fatalf("unexpected query result count: %d", cnt)
|
||||
}
|
||||
|
||||
// delete
|
||||
err = db.Delete("A")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// check if its gone
|
||||
_, err = db.Get("A")
|
||||
if err == nil {
|
||||
t.Fatal("should fail")
|
||||
}
|
||||
|
||||
// maintenance
|
||||
maintainer, ok := db.(storage.Maintainer)
|
||||
if ok {
|
||||
err = maintainer.Maintain(context.TODO())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = maintainer.MaintainThorough(context.TODO())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
t.Fatal("should implement Maintainer")
|
||||
}
|
||||
|
||||
// shutdown
|
||||
err = db.Shutdown()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user