Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
2024-08-09 17:15:48 +02:00
committed by GitHub
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions

View File

@@ -10,9 +10,9 @@ import (
"github.com/ghodss/yaml"
"github.com/safing/portbase/api"
"github.com/safing/portbase/log"
"github.com/safing/portbase/utils"
"github.com/safing/portmaster/base/api"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/utils"
)
const (
@@ -29,9 +29,8 @@ func registerAPIEndpoints() error {
Value: "",
Description: "Force downloading and applying of all updates, regardless of auto-update settings.",
}},
Path: apiPathCheckForUpdates,
Write: api.PermitUser,
BelongsTo: module,
Path: apiPathCheckForUpdates,
Write: api.PermitUser,
ActionFunc: func(r *api.Request) (msg string, err error) {
// Check if we should also download regardless of settings.
downloadAll := r.URL.Query().Has("download")
@@ -58,7 +57,6 @@ func registerAPIEndpoints() error {
Path: `updates/get/{identifier:[A-Za-z0-9/\.\-_]{1,255}}`,
Read: api.PermitUser,
ReadMethod: http.MethodGet,
BelongsTo: module,
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
// Get identifier from URL.
var identifier string

View File

@@ -1,12 +1,11 @@
package updates
import (
"context"
"github.com/tevino/abool"
"github.com/safing/portbase/config"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/updates/helper"
)
@@ -123,7 +122,7 @@ func initConfig() {
previousDevMode = devMode()
}
func updateRegistryConfig(_ context.Context, _ interface{}) error {
func updateRegistryConfig(_ *mgr.WorkerCtx, _ struct{}) (cancel bool, err error) {
changed := false
if enableSoftwareUpdates() != softwareUpdatesCurrentlyEnabled {
@@ -162,10 +161,10 @@ func updateRegistryConfig(_ context.Context, _ interface{}) error {
// Select versions depending on new indexes and modes.
registry.SelectVersions()
module.TriggerEvent(VersionUpdateEvent, nil)
module.EventVersionsUpdated.Submit(struct{}{})
if softwareUpdatesCurrentlyEnabled || intelUpdatesCurrentlyEnabled {
module.Resolve("")
module.states.Clear()
if err := TriggerUpdate(true, false); err != nil {
log.Warningf("updates: failed to trigger update: %s", err)
}
@@ -175,5 +174,5 @@ func updateRegistryConfig(_ context.Context, _ interface{}) error {
}
}
return nil
return false, nil
}

View File

@@ -1,17 +1,17 @@
package updates
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/info"
"github.com/safing/portbase/log"
"github.com/safing/portbase/updater"
"github.com/safing/portbase/utils/debug"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/info"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/updater"
"github.com/safing/portmaster/base/utils/debug"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/updates/helper"
)
@@ -152,12 +152,8 @@ func initVersionExport() (err error) {
log.Warningf("updates: failed to export version information: %s", err)
}
return module.RegisterEventHook(
ModuleName,
VersionUpdateEvent,
"export version status",
export,
)
module.EventVersionsUpdated.AddCallback("export version status", export)
return nil
}
func (v *Versions) save() error {
@@ -182,20 +178,20 @@ func (s *UpdateStateExport) save() error {
}
// export is an event hook.
func export(_ context.Context, _ interface{}) error {
func export(_ *mgr.WorkerCtx, _ struct{}) (cancel bool, err error) {
// Export versions.
if err := GetVersions().save(); err != nil {
return err
return false, err
}
if err := GetSimpleVersions().save(); err != nil {
return err
return false, err
}
// Export udpate state.
if err := GetStateExport().save(); err != nil {
return err
return false, err
}
return nil
return false, nil
}
// AddToDebugInfo adds the update system status to the given debug.Info.

View File

@@ -3,7 +3,7 @@ package updates
import (
"path"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/updater"
"github.com/safing/portmaster/service/updates/helper"
)
@@ -16,7 +16,7 @@ func GetPlatformFile(identifier string) (*updater.File, error) {
return nil, err
}
module.TriggerEvent(VersionUpdateEvent, nil)
module.EventVersionsUpdated.Submit(struct{}{})
return file, nil
}
@@ -29,7 +29,7 @@ func GetFile(identifier string) (*updater.File, error) {
return nil, err
}
module.TriggerEvent(VersionUpdateEvent, nil)
module.EventVersionsUpdated.Submit(struct{}{})
return file, nil
}

View File

@@ -8,8 +8,8 @@ import (
"runtime"
"strings"
"github.com/safing/portbase/log"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/updater"
)
var pmElectronUpdate *updater.File

View File

@@ -8,7 +8,7 @@ import (
"path/filepath"
"github.com/safing/jess/filesig"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/updater"
)
// Release Channel Configuration Keys.

View File

@@ -2,7 +2,7 @@ package helper
import (
"github.com/safing/jess"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/updater"
)
var (

View File

@@ -9,11 +9,11 @@ import (
"runtime"
"time"
"github.com/safing/portbase/database"
"github.com/safing/portbase/dataroot"
"github.com/safing/portbase/log"
"github.com/safing/portbase/modules"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/database"
"github.com/safing/portmaster/base/dataroot"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/updater"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/updates/helper"
)
@@ -43,13 +43,11 @@ const (
)
var (
module *modules.Module
registry *updater.ResourceRegistry
userAgentFromFlag string
updateServerFromFlag string
updateTask *modules.Task
updateASAP bool
disableTaskSchedule bool
@@ -80,19 +78,11 @@ const (
)
func init() {
module = modules.Register(ModuleName, prep, start, stop, "base")
module.RegisterEvent(VersionUpdateEvent, true)
module.RegisterEvent(ResourceUpdateEvent, true)
flag.StringVar(&updateServerFromFlag, "update-server", "", "set an alternative update server (full URL)")
flag.StringVar(&userAgentFromFlag, "update-agent", "", "set an alternative user agent for requests to the update server")
}
func prep() error {
if err := registerConfig(); err != nil {
return err
}
// Check if update server URL supplied via flag is a valid URL.
if updateServerFromFlag != "" {
u, err := url.Parse(updateServerFromFlag)
@@ -104,21 +94,18 @@ func prep() error {
}
}
if err := registerConfig(); err != nil {
return err
}
return registerAPIEndpoints()
}
func start() error {
initConfig()
restartTask = module.NewTask("automatic restart", automaticRestart).MaxDelay(10 * time.Minute)
if err := module.RegisterEventHook(
"config",
"config change",
"update registry config",
updateRegistryConfig); err != nil {
return err
}
module.restartWorkerMgr.Repeat(10 * time.Minute)
module.instance.Config().EventConfigChange.AddCallback("update registry config", updateRegistryConfig)
// create registry
registry = &updater.ResourceRegistry{
@@ -175,7 +162,7 @@ func start() error {
log.Warningf("updates: %s", warning)
}
err = registry.LoadIndexes(module.Ctx)
err = registry.LoadIndexes(module.m.Ctx())
if err != nil {
log.Warningf("updates: failed to load indexes: %s", err)
}
@@ -186,7 +173,7 @@ func start() error {
}
registry.SelectVersions()
module.TriggerEvent(VersionUpdateEvent, nil)
module.EventVersionsUpdated.Submit(struct{}{})
// Initialize the version export - this requires the registry to be set up.
err = initVersionExport()
@@ -195,18 +182,12 @@ func start() error {
}
// start updater task
updateTask = module.NewTask("updater", func(ctx context.Context, task *modules.Task) error {
return checkForUpdates(ctx)
})
if !disableTaskSchedule {
updateTask.
Repeat(updateTaskRepeatDuration).
MaxDelay(30 * time.Minute)
_ = module.updateWorkerMgr.Repeat(30 * time.Minute)
}
if updateASAP {
updateTask.StartASAP()
module.updateWorkerMgr.Go()
}
// react to upgrades
@@ -222,9 +203,6 @@ func start() error {
// TriggerUpdate queues the update task to execute ASAP.
func TriggerUpdate(forceIndexCheck, downloadAll bool) error {
switch {
case !module.Online():
updateASAP = true
case !forceIndexCheck && !enableSoftwareUpdates() && !enableIntelUpdates():
return errors.New("automatic updating is disabled")
@@ -237,11 +215,7 @@ func TriggerUpdate(forceIndexCheck, downloadAll bool) error {
}
// If index check if forced, start quicker.
if forceIndexCheck {
updateTask.StartASAP()
} else {
updateTask.Queue()
}
module.updateWorkerMgr.Go()
}
log.Debugf("updates: triggering update to run as soon as possible")
@@ -252,17 +226,18 @@ func TriggerUpdate(forceIndexCheck, downloadAll bool) error {
// If called, updates are only checked when TriggerUpdate()
// is called.
func DisableUpdateSchedule() error {
switch module.Status() {
case modules.StatusStarting, modules.StatusOnline, modules.StatusStopping:
return errors.New("module already online")
}
// TODO: Updater state should be always on
// switch module.Status() {
// case modules.StatusStarting, modules.StatusOnline, modules.StatusStopping:
// return errors.New("module already online")
// }
disableTaskSchedule = true
return nil
}
func checkForUpdates(ctx context.Context) (err error) {
func checkForUpdates(ctx *mgr.WorkerCtx) (err error) {
// Set correct error if context was canceled.
defer func() {
select {
@@ -295,12 +270,12 @@ func checkForUpdates(ctx context.Context) (err error) {
notifyUpdateCheckFailed(forceIndexCheck, err)
}()
if err = registry.UpdateIndexes(ctx); err != nil {
if err = registry.UpdateIndexes(ctx.Ctx()); err != nil {
err = fmt.Errorf("failed to update indexes: %w", err)
return //nolint:nakedret // TODO: Would "return err" work with the defer?
}
err = registry.DownloadUpdates(ctx, downloadAll)
err = registry.DownloadUpdates(ctx.Ctx(), downloadAll)
if err != nil {
err = fmt.Errorf("failed to download updates: %w", err)
return //nolint:nakedret // TODO: Would "return err" work with the defer?
@@ -318,7 +293,7 @@ func checkForUpdates(ctx context.Context) (err error) {
// Purge old resources
registry.Purge(2)
module.TriggerEvent(ResourceUpdateEvent, nil)
module.EventResourcesUpdated.Submit(struct{}{})
return nil
}
@@ -335,9 +310,9 @@ func stop() error {
// RootPath returns the root path used for storing updates.
func RootPath() string {
if !module.Online() {
return ""
}
// if !module.Online() {
// return ""
// }
return registry.StorageDir().Path
}

82
service/updates/module.go Normal file
View File

@@ -0,0 +1,82 @@
package updates
import (
"errors"
"sync/atomic"
"github.com/safing/portmaster/base/api"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/notifications"
"github.com/safing/portmaster/service/mgr"
)
// Updates provides access to released artifacts.
type Updates struct {
m *mgr.Manager
states *mgr.StateMgr
updateWorkerMgr *mgr.WorkerMgr
restartWorkerMgr *mgr.WorkerMgr
EventResourcesUpdated *mgr.EventMgr[struct{}]
EventVersionsUpdated *mgr.EventMgr[struct{}]
instance instance
}
var (
module *Updates
shimLoaded atomic.Bool
)
// New returns a new UI module.
func New(instance instance) (*Updates, error) {
if !shimLoaded.CompareAndSwap(false, true) {
return nil, errors.New("only one instance allowed")
}
m := mgr.New("Updates")
module = &Updates{
m: m,
states: m.NewStateMgr(),
updateWorkerMgr: m.NewWorkerMgr("updater", checkForUpdates, nil),
restartWorkerMgr: m.NewWorkerMgr("automatic restart", automaticRestart, nil),
EventResourcesUpdated: mgr.NewEventMgr[struct{}](ResourceUpdateEvent, m),
EventVersionsUpdated: mgr.NewEventMgr[struct{}](VersionUpdateEvent, m),
instance: instance,
}
if err := prep(); err != nil {
return nil, err
}
return module, nil
}
// States returns the state manager.
func (u *Updates) States() *mgr.StateMgr {
return u.states
}
// Manager returns the module manager.
func (u *Updates) Manager() *mgr.Manager {
return u.m
}
// Start starts the module.
func (u *Updates) Start() error {
return start()
}
// Stop stops the module.
func (u *Updates) Stop() error {
return stop()
}
type instance interface {
API() *api.API
Config() *config.Config
Restart()
Shutdown()
Notifications() *notifications.Notifications
}

View File

@@ -6,7 +6,7 @@ import (
"sync/atomic"
"time"
"github.com/safing/portbase/notifications"
"github.com/safing/portmaster/base/notifications"
)
const (
@@ -21,9 +21,17 @@ const (
var updateFailedCnt = new(atomic.Int32)
func (u *Updates) notificationsEnabled() bool {
return u.instance.Notifications() != nil
}
func notifyUpdateSuccess(force bool) {
if !module.notificationsEnabled() {
return
}
updateFailedCnt.Store(0)
module.Resolve(updateFailed)
module.states.Clear()
updateState := registry.GetState().Updates
flavor := updateSuccess
@@ -133,6 +141,10 @@ func getUpdatingInfoMsg() string {
}
func notifyUpdateCheckFailed(force bool, err error) {
if !module.notificationsEnabled() {
return
}
failedCnt := updateFailedCnt.Add(1)
lastSuccess := registry.GetState().Updates.LastSuccessAt
@@ -164,5 +176,5 @@ func notifyUpdateCheckFailed(force bool, err error) {
ResultAction: "display",
},
},
).AttachToModule(module)
).SyncWithState(module.states)
}

View File

@@ -15,9 +15,9 @@ import (
"github.com/tevino/abool"
"golang.org/x/exp/slices"
"github.com/safing/portbase/dataroot"
"github.com/safing/portbase/log"
"github.com/safing/portbase/utils/renameio"
"github.com/safing/portmaster/base/dataroot"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/utils/renameio"
)
var (

View File

@@ -1,7 +1,6 @@
package updates
import (
"context"
"os/exec"
"runtime"
"sync"
@@ -9,13 +8,8 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/log"
"github.com/safing/portbase/modules"
)
const (
// RestartExitCode will instruct portmaster-start to restart the process immediately, potentially with a new version.
RestartExitCode = 23
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
)
var (
@@ -23,7 +17,6 @@ var (
// should be restarted automatically when triggering a restart internally.
RebootOnRestart bool
restartTask *modules.Task
restartPending = abool.New()
restartTriggered = abool.New()
@@ -61,7 +54,7 @@ func DelayedRestart(delay time.Duration) {
// Schedule the restart task.
log.Warningf("updates: restart triggered, will execute in %s", delay)
restartAt := time.Now().Add(delay)
restartTask.Schedule(restartAt)
module.restartWorkerMgr.Delay(delay)
// Set restartTime.
restartTimeLock.Lock()
@@ -75,7 +68,7 @@ func AbortRestart() {
log.Warningf("updates: restart aborted")
// Cancel schedule.
restartTask.Schedule(time.Time{})
module.restartWorkerMgr.Delay(0)
}
}
@@ -83,7 +76,7 @@ func AbortRestart() {
// This can be used to prepone a scheduled restart if the conditions are preferable.
func TriggerRestartIfPending() {
if restartPending.IsSet() {
restartTask.StartASAP()
module.restartWorkerMgr.Go()
}
}
@@ -91,10 +84,10 @@ func TriggerRestartIfPending() {
// This only works if the process is managed by portmaster-start.
func RestartNow() {
restartPending.Set()
restartTask.StartASAP()
module.restartWorkerMgr.Go()
}
func automaticRestart(_ context.Context, _ *modules.Task) error {
func automaticRestart(w *mgr.WorkerCtx) error {
// Check if the restart is still scheduled.
if restartPending.IsNotSet() {
return nil
@@ -116,11 +109,10 @@ func automaticRestart(_ context.Context, _ *modules.Task) error {
// Set restart exit code.
if !rebooting {
modules.SetExitStatusCode(RestartExitCode)
module.instance.Restart()
} else {
module.instance.Shutdown()
}
// Do not use a worker, as this would block itself here.
go modules.Shutdown() //nolint:errcheck
}
return nil

View File

@@ -1,9 +1,9 @@
package updates
import (
"github.com/safing/portbase/database/record"
"github.com/safing/portbase/runtime"
"github.com/safing/portbase/updater"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/runtime"
"github.com/safing/portmaster/base/updater"
)
var pushRegistryStatusUpdate runtime.PushFunc

View File

@@ -14,13 +14,14 @@ import (
processInfo "github.com/shirou/gopsutil/process"
"github.com/tevino/abool"
"github.com/safing/portbase/dataroot"
"github.com/safing/portbase/info"
"github.com/safing/portbase/log"
"github.com/safing/portbase/notifications"
"github.com/safing/portbase/rng"
"github.com/safing/portbase/updater"
"github.com/safing/portbase/utils/renameio"
"github.com/safing/portmaster/base/dataroot"
"github.com/safing/portmaster/base/info"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/notifications"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/base/updater"
"github.com/safing/portmaster/base/utils/renameio"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/service/updates/helper"
)
@@ -41,23 +42,19 @@ var (
)
func initUpgrader() error {
return module.RegisterEventHook(
ModuleName,
ResourceUpdateEvent,
"run upgrades",
upgrader,
)
module.EventResourcesUpdated.AddCallback("run upgrades", upgrader)
return nil
}
func upgrader(_ context.Context, _ interface{}) error {
func upgrader(m *mgr.WorkerCtx, _ struct{}) (cancel bool, err error) {
// Lock runs, but discard additional runs.
if !upgraderActive.SetToIf(false, true) {
return nil
return false, nil
}
defer upgraderActive.SetTo(false)
// Upgrade portmaster-start.
err := upgradePortmasterStart()
err = upgradePortmasterStart()
if err != nil {
log.Warningf("updates: failed to upgrade portmaster-start: %s", err)
}
@@ -86,7 +83,7 @@ func upgrader(_ context.Context, _ interface{}) error {
}
}
return nil
return false, nil
}
func upgradeCoreNotify() error {
@@ -185,14 +182,14 @@ func upgradeHub() error {
// Increase update checks in order to detect aborts better.
if !disableTaskSchedule {
updateTask.Repeat(10 * time.Minute)
module.updateWorkerMgr.Repeat(10 * time.Minute)
}
} else {
AbortRestart()
// Set update task schedule back to normal.
if !disableTaskSchedule {
updateTask.Repeat(updateTaskRepeatDuration)
module.updateWorkerMgr.Repeat(updateTaskRepeatDuration)
}
}