Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
2024-08-09 17:15:48 +02:00
committed by GitHub
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions

View File

@@ -1,9 +1,10 @@
package profile
import (
"context"
"sync"
"time"
"github.com/safing/portmaster/service/mgr"
)
const (
@@ -53,7 +54,7 @@ func addActiveProfile(profile *Profile) {
activeProfiles[profile.ScopedID()] = profile
}
func cleanActiveProfiles(ctx context.Context) error {
func cleanActiveProfiles(ctx *mgr.WorkerCtx) error {
for {
select {
case <-time.After(activeProfileCleanerTickDuration):

View File

@@ -7,10 +7,10 @@ import (
"path/filepath"
"strings"
"github.com/safing/portbase/api"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portbase/utils"
"github.com/safing/portmaster/base/api"
"github.com/safing/portmaster/base/utils"
"github.com/safing/portmaster/service/profile/binmeta"
"github.com/safing/structures/dsd"
)
func registerAPIEndpoints() error {
@@ -19,7 +19,6 @@ func registerAPIEndpoints() error {
Description: "Merge multiple profiles into a new one.",
Path: "profile/merge",
Write: api.PermitUser,
BelongsTo: module,
StructFunc: handleMergeProfiles,
}); err != nil {
return err
@@ -30,7 +29,6 @@ func registerAPIEndpoints() error {
Description: "Returns the requested profile icon.",
Path: "profile/icon/{id:[a-f0-9]*\\.[a-z]{3,4}}",
Read: api.PermitUser,
BelongsTo: module,
DataFunc: handleGetProfileIcon,
}); err != nil {
return err
@@ -41,7 +39,6 @@ func registerAPIEndpoints() error {
Description: "Updates a profile icon.",
Path: "profile/icon",
Write: api.PermitUser,
BelongsTo: module,
StructFunc: handleUpdateProfileIcon,
}); err != nil {
return err

View File

@@ -9,8 +9,8 @@ import (
"github.com/vincent-petithory/dataurl"
"golang.org/x/exp/slices"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/record"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/database/record"
)
// Icon describes an icon.

View File

@@ -11,7 +11,7 @@ import (
"path/filepath"
"strings"
"github.com/safing/portbase/api"
"github.com/safing/portmaster/base/api"
)
// ProfileIconStoragePath defines the location where profile icons are stored.

View File

@@ -6,8 +6,8 @@ import (
"sync"
"time"
"github.com/safing/portbase/modules"
"github.com/safing/portmaster/service/intel/filterlists"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/profile/endpoints"
)
@@ -24,19 +24,16 @@ var (
)
func registerConfigUpdater() error {
return module.RegisterEventHook(
"config",
"config change",
"update global config profile",
func(ctx context.Context, _ interface{}) error {
return updateGlobalConfigProfile(ctx, nil)
},
)
module.instance.Config().EventConfigChange.AddCallback("update global config profile", func(wc *mgr.WorkerCtx, s struct{}) (cancel bool, err error) {
return false, updateGlobalConfigProfile(wc.Ctx())
})
return nil
}
const globalConfigProfileErrorID = "profile:global-profile-error"
func updateGlobalConfigProfile(ctx context.Context, task *modules.Task) error {
func updateGlobalConfigProfile(_ context.Context) error {
cfgLock.Lock()
defer cfgLock.Unlock()
@@ -132,25 +129,23 @@ func updateGlobalConfigProfile(ctx context.Context, task *modules.Task) error {
// If there was any error, try again later until it succeeds.
if lastErr == nil {
module.Resolve(globalConfigProfileErrorID)
module.states.Remove(globalConfigProfileErrorID)
} else {
// Create task after first failure.
if task == nil {
task = module.NewTask(
"retry updating global config profile",
updateGlobalConfigProfile,
)
}
// Schedule task.
task.Schedule(time.Now().Add(15 * time.Second))
_ = module.mgr.Delay("retry updating global config profile", 15*time.Second,
func(w *mgr.WorkerCtx) error {
return updateGlobalConfigProfile(w.Ctx())
})
// Add module warning to inform user.
module.Warning(
globalConfigProfileErrorID,
"Internal Settings Failure",
fmt.Sprintf("Some global settings might not be applied correctly. You can try restarting the Portmaster to resolve this problem. Error: %s", err),
)
module.states.Add(mgr.State{
ID: globalConfigProfileErrorID,
Name: "Internal Settings Failure",
Message: fmt.Sprintf("Some global settings might not be applied correctly. You can try restarting the Portmaster to resolve this problem. Error: %s", err),
Type: mgr.StateTypeWarning,
})
}
return lastErr

View File

@@ -3,7 +3,7 @@ package profile
import (
"strings"
"github.com/safing/portbase/config"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/service/profile/endpoints"
"github.com/safing/portmaster/service/status"
"github.com/safing/portmaster/spn/access/account"

View File

@@ -1,15 +1,15 @@
package profile
import (
"context"
"errors"
"strings"
"github.com/safing/portbase/config"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/query"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/database/query"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
)
// Database paths:
@@ -40,7 +40,7 @@ func registerValidationDBHook() (err error) {
}
func startProfileUpdateChecker() error {
module.StartServiceWorker("update active profiles", 0, func(ctx context.Context) (err error) {
module.mgr.Go("update active profiles", func(ctx *mgr.WorkerCtx) (err error) {
profilesSub, err := profileDB.Subscribe(query.New(ProfilesDBPath))
if err != nil {
return err
@@ -85,7 +85,7 @@ func startProfileUpdateChecker() error {
activeProfile.outdated.Set()
meta.MarkDeleted(scopedID)
module.TriggerEvent(DeletedEvent, scopedID)
module.EventDelete.Submit(scopedID)
continue
}
@@ -94,7 +94,7 @@ func startProfileUpdateChecker() error {
receivedProfile, err := EnsureProfile(r)
if err != nil || !receivedProfile.savedInternally {
activeProfile.outdated.Set()
module.TriggerEvent(ConfigChangeEvent, scopedID)
module.EventConfigChange.Submit(scopedID)
}
case <-ctx.Done():
return nil

View File

@@ -2,18 +2,117 @@ package endpoints
import (
"context"
"fmt"
"net"
"os"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/safing/portmaster/service/core/pmtesting"
"github.com/safing/portmaster/base/api"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database/dbmodule"
"github.com/safing/portmaster/base/notifications"
"github.com/safing/portmaster/service/intel"
"github.com/safing/portmaster/service/intel/geoip"
"github.com/safing/portmaster/service/updates"
)
type testInstance struct {
db *dbmodule.DBModule
api *api.API
config *config.Config
updates *updates.Updates
geoip *geoip.GeoIP
}
func (stub *testInstance) Updates() *updates.Updates {
return stub.updates
}
func (stub *testInstance) API() *api.API {
return stub.api
}
func (stub *testInstance) Config() *config.Config {
return stub.config
}
func (stub *testInstance) Notifications() *notifications.Notifications {
return nil
}
func (stub *testInstance) Ready() bool {
return true
}
func (stub *testInstance) Restart() {}
func (stub *testInstance) Shutdown() {}
func (stub *testInstance) SetCmdLineOperation(f func() error) {}
func runTest(m *testing.M) error {
api.SetDefaultAPIListenAddress("0.0.0.0:8080")
ds, err := config.InitializeUnitTestDataroot("test-endpoints")
if err != nil {
return fmt.Errorf("failed to initialize dataroot: %w", err)
}
defer func() { _ = os.RemoveAll(ds) }()
stub := &testInstance{}
stub.db, err = dbmodule.New(stub)
if err != nil {
return fmt.Errorf("failed to create database: %w", err)
}
stub.config, err = config.New(stub)
if err != nil {
return fmt.Errorf("failed to create config: %w", err)
}
stub.api, err = api.New(stub)
if err != nil {
return fmt.Errorf("failed to create api: %w", err)
}
stub.updates, err = updates.New(stub)
if err != nil {
return fmt.Errorf("failed to create updates: %w", err)
}
stub.geoip, err = geoip.New(stub)
if err != nil {
return fmt.Errorf("failed to create geoip: %w", err)
}
err = stub.db.Start()
if err != nil {
return fmt.Errorf("Failed to start database: %w", err)
}
err = stub.config.Start()
if err != nil {
return fmt.Errorf("Failed to start config: %w", err)
}
err = stub.api.Start()
if err != nil {
return fmt.Errorf("Failed to start api: %w", err)
}
err = stub.updates.Start()
if err != nil {
return fmt.Errorf("Failed to start updates: %w", err)
}
err = stub.geoip.Start()
if err != nil {
return fmt.Errorf("Failed to start geoip: %w", err)
}
m.Run()
return nil
}
func TestMain(m *testing.M) {
pmtesting.TestMain(m, intel.Module)
if err := runTest(m); err != nil {
fmt.Printf("%s", err)
os.Exit(1)
}
}
func testEndpointMatch(t *testing.T, ep Endpoint, entity *intel.Entity, expectedResult EPResult) {

View File

@@ -8,7 +8,7 @@ import (
"golang.org/x/exp/slices"
"github.com/safing/jess/lhash"
"github.com/safing/portbase/container"
"github.com/safing/structures/container"
)
// # Matching and Scores

View File

@@ -40,7 +40,7 @@ func TestDeriveProfileID(t *testing.T) {
rnd := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
// Test 100 times.
for i := 0; i < 100; i++ {
for range 100 {
// Shuffle fingerprints.
rnd.Shuffle(len(fps), func(i, j int) {
fps[i], fps[j] = fps[j], fps[i]

View File

@@ -9,7 +9,7 @@ package profile
// "regexp"
// "strings"
//
// "github.com/safing/portbase/log"
// "github.com/safing/portmaster/base/log"
// )
//
// type Framework struct {

View File

@@ -1,18 +1,18 @@
package profile
import (
"context"
"errors"
"fmt"
"path"
"strings"
"sync"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/query"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/log"
"github.com/safing/portbase/notifications"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/database/query"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/notifications"
"github.com/safing/portmaster/service/mgr"
)
var getProfileLock sync.Mutex
@@ -147,8 +147,8 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P
// Trigger further metadata fetching from system if profile was created.
if created && profile.UsePresentationPath && !special {
module.StartWorker("get profile metadata", func(ctx context.Context) error {
return profile.updateMetadataFromSystem(ctx, md)
module.mgr.Go("get profile metadata", func(wc *mgr.WorkerCtx) error {
return profile.updateMetadataFromSystem(wc.Ctx(), md)
})
}

View File

@@ -6,7 +6,7 @@ import (
"sync"
"time"
"github.com/safing/portbase/database/record"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/service/profile/binmeta"
)
@@ -77,12 +77,12 @@ func MergeProfiles(name string, primary *Profile, secondaries ...*Profile) (newP
if err := primary.delete(); err != nil {
return nil, fmt.Errorf("failed to delete primary profile %s: %w", primary.ScopedID(), err)
}
module.TriggerEvent(MigratedEvent, []string{primary.ScopedID(), newProfile.ScopedID()})
module.EventMigrated.Submit([]string{primary.ScopedID(), newProfile.ScopedID()})
for _, sp := range secondaries {
if err := sp.delete(); err != nil {
return nil, fmt.Errorf("failed to delete secondary profile %s: %w", sp.ScopedID(), err)
}
module.TriggerEvent(MigratedEvent, []string{sp.ScopedID(), newProfile.ScopedID()})
module.EventMigrated.Submit([]string{sp.ScopedID(), newProfile.ScopedID()})
}
return newProfile, nil

View File

@@ -5,7 +5,7 @@ import (
"sync"
"time"
"github.com/safing/portbase/database/record"
"github.com/safing/portmaster/base/database/record"
)
// ProfilesMetadata holds metadata about all profiles that are not fit to be

View File

@@ -7,10 +7,11 @@ import (
"github.com/hashicorp/go-version"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/migration"
"github.com/safing/portbase/database/query"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/database/migration"
"github.com/safing/portmaster/base/database/query"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/profile/binmeta"
)
@@ -129,11 +130,12 @@ func migrateIcons(ctx context.Context, _, to *version.Version, db *database.Inte
if lastErr != nil {
// Normally, an icon migration would not be such a big error, but this is a test
// run for the profile IDs and we absolutely need to know if anything went wrong.
module.Error(
"migration-failed",
"Profile Migration Failed",
fmt.Sprintf("Failed to migrate icons of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
)
module.states.Add(mgr.State{
ID: "migration-failed-icons",
Name: "Profile Migration Failed",
Message: fmt.Sprintf("Failed to migrate icons of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
Type: mgr.StateTypeError,
})
return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr)
}
@@ -217,11 +219,12 @@ func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *databa
// Log migration failure and try again next time.
if lastErr != nil {
module.Error(
"migration-failed",
"Profile Migration Failed",
fmt.Sprintf("Failed to migrate profile IDs of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
)
module.states.Add(mgr.State{
ID: "migration-failed-derived-IDs",
Name: "Profile Migration Failed",
Message: fmt.Sprintf("Failed to migrate profile IDs of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
Type: mgr.StateTypeError,
})
return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr)
}

View File

@@ -4,20 +4,21 @@ import (
"errors"
"fmt"
"os"
"sync/atomic"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/migration"
"github.com/safing/portbase/dataroot"
"github.com/safing/portbase/log"
"github.com/safing/portbase/modules"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/database/migration"
"github.com/safing/portmaster/base/dataroot"
"github.com/safing/portmaster/base/log"
_ "github.com/safing/portmaster/service/core/base"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/profile/binmeta"
"github.com/safing/portmaster/service/updates"
)
var (
migrations = migration.New("core:migrations/profile")
module *modules.Module
updatesPath string
)
@@ -28,11 +29,31 @@ const (
MigratedEvent = "profile migrated"
)
func init() {
module = modules.Register("profiles", prep, start, stop, "base", "updates")
module.RegisterEvent(ConfigChangeEvent, true)
module.RegisterEvent(DeletedEvent, true)
module.RegisterEvent(MigratedEvent, true)
type ProfileModule struct {
mgr *mgr.Manager
instance instance
EventConfigChange *mgr.EventMgr[string]
EventDelete *mgr.EventMgr[string]
EventMigrated *mgr.EventMgr[[]string]
states *mgr.StateMgr
}
func (pm *ProfileModule) Manager() *mgr.Manager {
return pm.mgr
}
func (pm *ProfileModule) States() *mgr.StateMgr {
return pm.states
}
func (pm *ProfileModule) Start() error {
return start()
}
func (pm *ProfileModule) Stop() error {
return stop()
}
func prep() error {
@@ -48,6 +69,10 @@ func prep() error {
return err
}
if err := registerAPIEndpoints(); err != nil {
return err
}
// Setup icon storage location.
iconsDir := dataroot.Root().ChildDir("databases", 0o0700).ChildDir("icons", 0o0700)
if err := iconsDir.Ensure(); err != nil {
@@ -72,7 +97,7 @@ func start() error {
}
meta.check()
if err := migrations.Migrate(module.Ctx); err != nil {
if err := migrations.Migrate(module.mgr.Ctx()); err != nil {
log.Errorf("profile: migrations failed: %s", err)
}
@@ -91,20 +116,48 @@ func start() error {
return err
}
module.StartServiceWorker("clean active profiles", 0, cleanActiveProfiles)
module.mgr.Go("clean active profiles", cleanActiveProfiles)
err = updateGlobalConfigProfile(module.Ctx, nil)
err = updateGlobalConfigProfile(module.mgr.Ctx())
if err != nil {
log.Warningf("profile: error during loading global profile from configuration: %s", err)
}
if err := registerAPIEndpoints(); err != nil {
return err
}
return nil
}
func stop() error {
return meta.Save()
}
var (
module *ProfileModule
shimLoaded atomic.Bool
)
func NewModule(instance instance) (*ProfileModule, error) {
if !shimLoaded.CompareAndSwap(false, true) {
return nil, errors.New("only one instance allowed")
}
m := mgr.New("Profile")
module = &ProfileModule{
mgr: m,
instance: instance,
EventConfigChange: mgr.NewEventMgr[string](ConfigChangeEvent, m),
EventDelete: mgr.NewEventMgr[string](DeletedEvent, m),
EventMigrated: mgr.NewEventMgr[[]string](MigratedEvent, m),
states: mgr.NewStateMgr(m),
}
if err := prep(); err != nil {
return nil, err
}
return module, nil
}
type instance interface {
Config() *config.Config
}

View File

@@ -4,9 +4,9 @@ import (
"errors"
"strings"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/log"
"github.com/safing/portbase/runtime"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/runtime"
)
const (

View File

@@ -5,10 +5,10 @@ import (
"sync"
"sync/atomic"
"github.com/safing/portbase/config"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/log"
"github.com/safing/portbase/runtime"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/runtime"
"github.com/safing/portmaster/service/intel"
"github.com/safing/portmaster/service/profile/endpoints"
)

View File

@@ -11,10 +11,10 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/config"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/log"
"github.com/safing/portbase/utils"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/utils"
"github.com/safing/portmaster/service/intel/filterlists"
"github.com/safing/portmaster/service/profile/binmeta"
"github.com/safing/portmaster/service/profile/endpoints"

View File

@@ -3,7 +3,7 @@ package profile
import (
"time"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/log"
)
const (