github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Restructure modules (#1572)

Browse Source 
* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
2024-08-09 17:15:48 +02:00
committed by GitHub
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
spn/docks/bandwidth_test.go
View File

@@ -6,9 +6,9 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/dsd"
)
func TestEffectiveBandwidth(t *testing.T) { //nolint:paralleltest // Run alone.
@@ -66,7 +66,7 @@ func TestEffectiveBandwidth(t *testing.T) { //nolint:paralleltest // Run alone.
t.Fatal(tErr)
}
// Start handler.
module.StartWorker("op capacity handler", op.handler)
module.mgr.Go("op capacity handler", op.handler)
// Wait for result and check error.
tErr = <-op.Result()

4
spn/docks/controller.go
View File

@@ -1,8 +1,8 @@
package docks
import (
"github.com/safing/portbase/container"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
)
// CraneControllerTerminal is a terminal for the crane itself.
@@ -80,7 +80,7 @@ func initCraneController(
t.GrantPermission(terminal.IsCraneController)
// Start workers.
t.StartWorkers(module, "crane controller terminal")
t.StartWorkers(module.mgr, "crane controller terminal")
return cct
}

23
spn/docks/crane.go
View File

@@ -12,14 +12,15 @@ import (
"github.com/tevino/abool"
"github.com/safing/jess"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/varint"
"github.com/safing/portbase/log"
"github.com/safing/portbase/rng"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/cabin"
"github.com/safing/portmaster/spn/hub"
"github.com/safing/portmaster/spn/ships"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/varint"
)
const (
@@ -110,7 +111,7 @@ type Crane struct {
// NewCrane returns a new crane.
func NewCrane(ship ships.Ship, connectedHub *hub.Hub, id *cabin.Identity) (*Crane, error) {
// Cranes always run in module context.
ctx, cancelCtx := context.WithCancel(module.Ctx)
ctx, cancelCtx := context.WithCancel(module.mgr.Ctx())
newCrane := &Crane{
ctx: ctx,
@@ -351,7 +352,7 @@ func (crane *Crane) AbandonTerminal(id uint32, err *terminal.Error) {
if crane.stopping.IsSet() &&
crane.terminalCount() <= 1 {
// Stop the crane in worker, so the caller can do some work.
module.StartWorker("retire crane", func(_ context.Context) error {
module.mgr.Go("retire crane", func(_ *mgr.WorkerCtx) error {
// Let enough time for the last errors to be sent, as terminals are abandoned in a goroutine.
time.Sleep(3 * time.Second)
crane.Stop(nil)
@@ -426,7 +427,7 @@ func (crane *Crane) decrypt(shipment *container.Container) (decrypted *container
return container.New(decryptedData), nil
}
func (crane *Crane) unloader(workerCtx context.Context) error {
func (crane *Crane) unloader(workerCtx *mgr.WorkerCtx) error {
// Unclean shutdown safeguard.
defer crane.Stop(terminal.ErrUnknownError.With("unloader died"))
@@ -516,7 +517,7 @@ func (crane *Crane) unloadUntilFull(buf []byte) error {
}
}
func (crane *Crane) handler(workerCtx context.Context) error {
func (crane *Crane) handler(workerCtx *mgr.WorkerCtx) error {
var partialShipment *container.Container
var segmentLength uint32
@@ -618,7 +619,7 @@ handling:
if deliveryErr != nil {
msg.Finish()
// This is a hot path. Start a worker for abandoning the terminal.
module.StartWorker("end terminal", func(_ context.Context) error {
module.mgr.Go("end terminal", func(_ *mgr.WorkerCtx) error {
crane.AbandonTerminal(t.ID(), deliveryErr.Wrap("failed to deliver data"))
return nil
})
@@ -635,7 +636,7 @@ handling:
receivedErr = terminal.ErrUnknownError.AsExternal()
}
// This is a hot path. Start a worker for abandoning the terminal.
module.StartWorker("end terminal", func(_ context.Context) error {
module.mgr.Go("end terminal", func(_ *mgr.WorkerCtx) error {
crane.AbandonTerminal(terminalID, receivedErr)
return nil
})
@@ -645,7 +646,7 @@ handling:
}
}
func (crane *Crane) loader(workerCtx context.Context) (err error) {
func (crane *Crane) loader(workerCtx *mgr.WorkerCtx) (err error) {
shipment := container.New()
var partialShipment *container.Container
var loadingTimer *time.Timer

8
spn/docks/crane_establish.go
View File

@@ -1,12 +1,12 @@
package docks
import (
"context"
"time"
"github.com/safing/portbase/container"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
)
const (
@@ -70,7 +70,7 @@ func (crane *Crane) establishTerminal(id uint32, initData *container.Container)
case crane.terminalMsgs <- msg:
default:
// Send error async.
module.StartWorker("abandon terminal", func(ctx context.Context) error {
module.mgr.Go("abandon terminal", func(ctx *mgr.WorkerCtx) error {
select {
case crane.terminalMsgs <- msg:
case <-ctx.Done():

22
spn/docks/crane_init.go
View File

@@ -5,13 +5,13 @@ import (
"time"
"github.com/safing/jess"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portbase/formats/varint"
"github.com/safing/portbase/info"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/info"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/spn/conf"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/dsd"
"github.com/safing/structures/varint"
)
/*
@@ -71,7 +71,7 @@ func (crane *Crane) Start(callerCtx context.Context) error {
}
func (crane *Crane) startLocal(callerCtx context.Context) *terminal.Error {
module.StartWorker("crane unloader", crane.unloader)
module.mgr.Go("crane unloader", crane.unloader)
if !crane.ship.IsSecure() {
// Start encrypted channel.
@@ -171,8 +171,8 @@ func (crane *Crane) startLocal(callerCtx context.Context) *terminal.Error {
}
// Start remaining workers.
module.StartWorker("crane loader", crane.loader)
module.StartWorker("crane handler", crane.handler)
module.mgr.Go("crane loader", crane.loader)
module.mgr.Go("crane handler", crane.handler)
return nil
}
@@ -180,7 +180,7 @@ func (crane *Crane) startLocal(callerCtx context.Context) *terminal.Error {
func (crane *Crane) startRemote(callerCtx context.Context) *terminal.Error {
var initMsg *container.Container
module.StartWorker("crane unloader", crane.unloader)
module.mgr.Go("crane unloader", crane.unloader)
handling:
for {
@@ -270,8 +270,8 @@ handling:
}
// Start remaining workers.
module.StartWorker("crane loader", crane.loader)
module.StartWorker("crane handler", crane.handler)
module.mgr.Go("crane loader", crane.loader)
module.mgr.Go("crane handler", crane.handler)
return nil
}

4
spn/docks/crane_terminal.go
View File

@@ -3,9 +3,9 @@ package docks
import (
"net"
"github.com/safing/portbase/container"
"github.com/safing/portmaster/spn/hub"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
)
// CraneTerminal is a terminal started by a crane.
@@ -74,7 +74,7 @@ func initCraneTerminal(
t.SetTerminalExtension(ct)
// Start workers.
t.StartWorkers(module, "crane terminal")
t.StartWorkers(module.mgr, "crane terminal")
return ct
}

12
spn/docks/crane_test.go
View File

@@ -47,7 +47,7 @@ func testCraneWithCounter(t *testing.T, testID string, encrypting bool, loadSize
if err != nil {
panic(fmt.Sprintf("crane test %s could not create crane1: %s", testID, err))
}
err = crane1.Start(module.Ctx)
err = crane1.Start(module.mgr.Ctx())
if err != nil {
panic(fmt.Sprintf("crane test %s could not start crane1: %s", testID, err))
}
@@ -59,7 +59,7 @@ func testCraneWithCounter(t *testing.T, testID string, encrypting bool, loadSize
if err != nil {
panic(fmt.Sprintf("crane test %s could not create crane2: %s", testID, err))
}
err = crane2.Start(module.Ctx)
err = crane2.Start(module.mgr.Ctx())
if err != nil {
panic(fmt.Sprintf("crane test %s could not start crane2: %s", testID, err))
}
@@ -122,7 +122,7 @@ func (t *StreamingTerminal) ID() uint32 {
}
func (t *StreamingTerminal) Ctx() context.Context {
return module.Ctx
return module.mgr.Ctx()
}
func (t *StreamingTerminal) Deliver(msg *terminal.Msg) *terminal.Error {
@@ -170,7 +170,7 @@ func testCraneWithStreaming(t *testing.T, testID string, encrypting bool, loadSi
if err != nil {
panic(fmt.Sprintf("crane test %s could not create crane1: %s", testID, err))
}
err = crane1.Start(module.Ctx)
err = crane1.Start(module.mgr.Ctx())
if err != nil {
panic(fmt.Sprintf("crane test %s could not start crane1: %s", testID, err))
}
@@ -182,7 +182,7 @@ func testCraneWithStreaming(t *testing.T, testID string, encrypting bool, loadSi
if err != nil {
panic(fmt.Sprintf("crane test %s could not create crane2: %s", testID, err))
}
err = crane2.Start(module.Ctx)
err = crane2.Start(module.mgr.Ctx())
if err != nil {
panic(fmt.Sprintf("crane test %s could not start crane2: %s", testID, err))
}
@@ -257,7 +257,7 @@ func getTestIdentity(t *testing.T) (*cabin.Identity, *hub.Hub) {
if testIdentity == nil {
var err error
testIdentity, err = cabin.CreateIdentity(module.Ctx, "test")
testIdentity, err = cabin.CreateIdentity(module.mgr.Ctx(), "test")
if err != nil {
t.Fatalf("failed to create identity: %s", err)
}

4
spn/docks/crane_verify.go
View File

@@ -6,10 +6,10 @@ import (
"fmt"
"time"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/varint"
"github.com/safing/portmaster/spn/cabin"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/varint"
)
const (

2
spn/docks/cranehooks.go
View File

@@ -3,7 +3,7 @@ package docks
import (
"sync"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/log"
)
var (

6
spn/docks/hub_import.go
View File

@@ -6,7 +6,7 @@ import (
"net"
"sync"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/spn/conf"
"github.com/safing/portmaster/spn/hub"
"github.com/safing/portmaster/spn/ships"
@@ -33,7 +33,7 @@ func ImportAndVerifyHubInfo(ctx context.Context, hubID string, announcementData,
var hubKnown, hubChanged bool
if announcementData != nil {
hubFromMsg, known, changed, err := hub.ApplyAnnouncement(nil, announcementData, mapName, scope, false)
if err != nil && firstErr == nil {
if err != nil {
firstErr = terminal.ErrInternalError.With("failed to apply announcement: %w", err)
}
if known {
@@ -170,7 +170,7 @@ func verifyHubIP(ctx context.Context, h *hub.Hub, ip net.IP) error {
if err != nil {
return fmt.Errorf("failed to create crane: %w", err)
}
module.StartWorker("crane unloader", crane.unloader)
module.mgr.Go("crane unloader", crane.unloader)
defer crane.Stop(nil)
// Verify Hub.

4
spn/docks/metrics.go
View File

@@ -7,8 +7,8 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/api"
"github.com/safing/portbase/metrics"
"github.com/safing/portmaster/base/api"
"github.com/safing/portmaster/base/metrics"
)
var (

54
spn/docks/module.go
View File

@@ -5,15 +5,35 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/safing/portbase/modules"
"github.com/safing/portbase/rng"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/service/mgr"
_ "github.com/safing/portmaster/spn/access"
)
var (
module *modules.Module
// Docks handles connections to other network participants.
type Docks struct {
mgr *mgr.Manager
instance instance
}
// Manager returns the module manager.
func (d *Docks) Manager() *mgr.Manager {
return d.mgr
}
// Start starts the module.
func (d *Docks) Start() error {
return start()
}
// Stop stops the module.
func (d *Docks) Stop() error {
return stopAllCranes()
}
var (
allCranes = make(map[string]*Crane) // ID = Crane ID
assignedCranes = make(map[string]*Crane) // ID = connected Hub ID
cranesLock sync.RWMutex
@@ -21,10 +41,6 @@ var (
runningTests bool
)
func init() {
module = modules.Register("docks", nil, start, stopAllCranes, "terminal", "cabin", "access")
}
func start() error {
return registerMetrics()
}
@@ -34,7 +50,7 @@ func registerCrane(crane *Crane) error {
defer cranesLock.Unlock()
// Generate new IDs until a unique one is found.
for i := 0; i < 100; i++ {
for range 100 {
// Generate random ID.
randomID, err := rng.Bytes(3)
if err != nil {
@@ -115,3 +131,23 @@ func GetAllAssignedCranes() map[string]*Crane {
}
return copiedCranes
}
var (
module *Docks
shimLoaded atomic.Bool
)
// New returns a new Docks module.
func New(instance instance) (*Docks, error) {
if !shimLoaded.CompareAndSwap(false, true) {
return nil, errors.New("only one instance allowed")
}
m := mgr.New("Docks")
module = &Docks{
mgr: m,
instance: instance,
}
return module, nil
}
type instance interface{}

138
spn/docks/module_test.go
View File

@@ -1,16 +1,148 @@
package docks
import (
"fmt"
"os"
"testing"
"github.com/safing/portmaster/service/core/pmtesting"
"github.com/safing/portmaster/base/config"
"github.com/safing/portmaster/base/database/dbmodule"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/metrics"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/service/core/base"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/access"
"github.com/safing/portmaster/spn/cabin"
"github.com/safing/portmaster/spn/conf"
"github.com/safing/portmaster/spn/terminal"
)
func TestMain(m *testing.M) {
type testInstance struct {
db *dbmodule.DBModule
config *config.Config
metrics *metrics.Metrics
rng *rng.Rng
base *base.Base
access *access.Access
terminal *terminal.TerminalModule
cabin *cabin.Cabin
}
func (stub *testInstance) Config() *config.Config {
return stub.config
}
func (stub *testInstance) Metrics() *metrics.Metrics {
return stub.metrics
}
func (stub *testInstance) SPNGroup() *mgr.ExtendedGroup {
return nil
}
func (stub *testInstance) Stopping() bool {
return false
}
func (stub *testInstance) SetCmdLineOperation(f func() error) {}
func runTest(m *testing.M) error {
_ = log.Start()
ds, err := config.InitializeUnitTestDataroot("test-docks")
if err != nil {
return fmt.Errorf("failed to initialize dataroot: %w", err)
}
defer func() { _ = os.RemoveAll(ds) }()
instance := &testInstance{}
runningTests = true
conf.EnablePublicHub(true) // Make hub config available.
access.EnableTestMode() // Register test zone instead of real ones.
pmtesting.TestMain(m, module)
// Init
instance.db, err = dbmodule.New(instance)
if err != nil {
return fmt.Errorf("failed to create database module: %w", err)
}
instance.config, err = config.New(instance)
if err != nil {
return fmt.Errorf("failed to create config module: %w", err)
}
instance.metrics, err = metrics.New(instance)
if err != nil {
return fmt.Errorf("failed to create metrics module: %w", err)
}
instance.rng, err = rng.New(instance)
if err != nil {
return fmt.Errorf("failed to create rng module: %w", err)
}
instance.base, err = base.New(instance)
if err != nil {
return fmt.Errorf("failed to create base module: %w", err)
}
instance.access, err = access.New(instance)
if err != nil {
return fmt.Errorf("failed to create access module: %w", err)
}
instance.terminal, err = terminal.New(instance)
if err != nil {
return fmt.Errorf("failed to create terminal module: %w", err)
}
instance.cabin, err = cabin.New(instance)
if err != nil {
return fmt.Errorf("failed to create cabin module: %w", err)
}
module, err = New(instance)
if err != nil {
return fmt.Errorf("failed to create docks module: %w", err)
}
// Start
err = instance.db.Start()
if err != nil {
return fmt.Errorf("failed to start db module: %w", err)
}
err = instance.config.Start()
if err != nil {
return fmt.Errorf("failed to start config module: %w", err)
}
err = instance.metrics.Start()
if err != nil {
return fmt.Errorf("failed to start metrics module: %w", err)
}
err = instance.rng.Start()
if err != nil {
return fmt.Errorf("failed to start rng module: %w", err)
}
err = instance.base.Start()
if err != nil {
return fmt.Errorf("failed to start base module: %w", err)
}
err = instance.access.Start()
if err != nil {
return fmt.Errorf("failed to start access module: %w", err)
}
err = instance.terminal.Start()
if err != nil {
return fmt.Errorf("failed to start terminal module: %w", err)
}
err = instance.cabin.Start()
if err != nil {
return fmt.Errorf("failed to start cabin module: %w", err)
}
err = module.Start()
if err != nil {
return fmt.Errorf("failed to start docks module: %w", err)
}
m.Run()
return nil
}
func TestMain(m *testing.M) {
if err := runTest(m); err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
}

20
spn/docks/op_capacity.go
View File

@@ -2,16 +2,16 @@ package docks
import (
"bytes"
"context"
"sync/atomic"
"time"
"github.com/tevino/abool"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portbase/log"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/dsd"
)
const (
@@ -116,7 +116,7 @@ func NewCapacityTestOp(t terminal.Terminal, opts *CapacityTestOptions) (*Capacit
}
// Start handler.
module.StartWorker("op capacity handler", op.handler)
module.mgr.Go("op capacity handler", op.handler)
return op, nil
}
@@ -157,13 +157,13 @@ func startCapacityTestOp(t terminal.Terminal, opID uint32, data *container.Conta
// Start handler and sender.
op.senderStarted = true
module.StartWorker("op capacity handler", op.handler)
module.StartWorker("op capacity sender", op.sender)
module.mgr.Go("op capacity handler", op.handler)
module.mgr.Go("op capacity sender", op.sender)
return op, nil
}
func (op *CapacityTestOp) handler(ctx context.Context) error {
func (op *CapacityTestOp) handler(ctx *mgr.WorkerCtx) error {
defer capacityTestRunning.UnSet()
returnErr := terminal.ErrStopping
@@ -204,7 +204,7 @@ func (op *CapacityTestOp) handler(ctx context.Context) error {
maxTestTimeReached = time.After(op.opts.MaxTime)
if !op.senderStarted {
op.senderStarted = true
module.StartWorker("op capacity sender", op.sender)
module.mgr.Go("op capacity sender", op.sender)
}
}
@@ -241,7 +241,7 @@ func (op *CapacityTestOp) handler(ctx context.Context) error {
}
}
func (op *CapacityTestOp) sender(ctx context.Context) error {
func (op *CapacityTestOp) sender(ctx *mgr.WorkerCtx) error {
for {
// Send next chunk.
msg := op.NewMsg(capacityTestSendData)

2
spn/docks/op_capacity_test.go
View File

@@ -60,7 +60,7 @@ func testCapacityOp(t *testing.T, opts *CapacityTestOptions) {
b.GrantPermission(terminal.IsCraneController)
op, tErr := NewCapacityTestOp(a, opts)
if tErr != nil {
t.Fatalf("failed to start op: %s", err)
t.Fatalf("failed to start op: %s", tErr)
}
// Wait for result and check error.

17
spn/docks/op_expand.go
View File

@@ -8,9 +8,10 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/container"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/conf"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
)
// ExpandOpType is the type ID of the expand operation.
@@ -201,13 +202,13 @@ func expand(t terminal.Terminal, opID uint32, data *container.Container) (termin
}
// Start workers.
module.StartWorker("expand op forward relay", op.forwardHandler)
module.StartWorker("expand op backward relay", op.backwardHandler)
module.mgr.Go("expand op forward relay", op.forwardHandler)
module.mgr.Go("expand op backward relay", op.backwardHandler)
if op.flowControl != nil {
op.flowControl.StartWorkers(module, "expand op")
op.flowControl.StartWorkers(module.mgr, "expand op")
}
if op.relayTerminal.flowControl != nil {
op.relayTerminal.flowControl.StartWorkers(module, "expand op terminal")
op.relayTerminal.flowControl.StartWorkers(module.mgr, "expand op terminal")
}
return op, nil
@@ -259,7 +260,7 @@ func (op *ExpandOp) submitBackwardUpstream(msg *terminal.Msg, timeout time.Durat
}
}
func (op *ExpandOp) forwardHandler(_ context.Context) error {
func (op *ExpandOp) forwardHandler(_ *mgr.WorkerCtx) error {
// Metrics setup and submitting.
atomic.AddInt64(activeExpandOps, 1)
started := time.Now()
@@ -290,7 +291,7 @@ func (op *ExpandOp) forwardHandler(_ context.Context) error {
}
}
func (op *ExpandOp) backwardHandler(_ context.Context) error {
func (op *ExpandOp) backwardHandler(_ *mgr.WorkerCtx) error {
for {
select {
case msg := <-op.relayTerminal.recvProxy():
@@ -336,7 +337,7 @@ func (op *ExpandOp) HandleStop(err *terminal.Error) (errorToSend *terminal.Error
// Abandon shuts down the terminal unregistering it from upstream and calling HandleAbandon().
func (t *ExpansionRelayTerminal) Abandon(err *terminal.Error) {
if t.abandoning.SetToIf(false, true) {
module.StartWorker("terminal abandon procedure", func(_ context.Context) error {
module.mgr.Go("terminal abandon procedure", func(_ *mgr.WorkerCtx) error {
t.handleAbandonProcedure(err)
return nil
})

14
spn/docks/op_latency.go
View File

@@ -2,15 +2,15 @@ package docks
import (
"bytes"
"context"
"fmt"
"time"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/varint"
"github.com/safing/portbase/log"
"github.com/safing/portbase/rng"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/varint"
)
const (
@@ -82,12 +82,12 @@ func NewLatencyTestOp(t terminal.Terminal) (*LatencyTestClientOp, *terminal.Erro
}
// Start handler.
module.StartWorker("op latency handler", op.handler)
module.mgr.Go("op latency handler", op.handler)
return op, nil
}
func (op *LatencyTestClientOp) handler(ctx context.Context) error {
func (op *LatencyTestClientOp) handler(ctx *mgr.WorkerCtx) error {
returnErr := terminal.ErrStopping
defer func() {
// Linters don't get that returnErr is used when directly used as defer.

2
spn/docks/op_latency_test.go
View File

@@ -35,7 +35,7 @@ func TestLatencyOp(t *testing.T) {
b.GrantPermission(terminal.IsCraneController)
op, tErr := NewLatencyTestOp(a)
if tErr != nil {
t.Fatalf("failed to start op: %s", err)
t.Fatalf("failed to start op: %s", tErr)
}
// Wait for result and check error.

9
spn/docks/op_sync_state.go
View File

@@ -4,10 +4,11 @@ import (
"context"
"time"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/portmaster/spn/conf"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/dsd"
)
// SyncStateOpType is the type ID of the sync state operation.
@@ -39,8 +40,8 @@ func init() {
// startSyncStateOp starts a worker that runs the sync state operation.
func (crane *Crane) startSyncStateOp() {
module.StartWorker("sync crane state", func(ctx context.Context) error {
tErr := crane.Controller.SyncState(ctx)
module.mgr.Go("sync crane state", func(wc *mgr.WorkerCtx) error {
tErr := crane.Controller.SyncState(wc.Ctx())
if tErr != nil {
return tErr
}

4
spn/docks/op_whoami.go
View File

@@ -3,9 +3,9 @@ package docks
import (
"time"
"github.com/safing/portbase/container"
"github.com/safing/portbase/formats/dsd"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
"github.com/safing/structures/dsd"
)
const (

6
spn/docks/terminal_expansion.go
View File

@@ -7,9 +7,9 @@ import (
"github.com/tevino/abool"
"github.com/safing/portbase/container"
"github.com/safing/portmaster/spn/hub"
"github.com/safing/portmaster/spn/terminal"
"github.com/safing/structures/container"
)
// ExpansionTerminal is used for expanding to another Hub.
@@ -53,7 +53,7 @@ func ExpandTo(from terminal.Terminal, routeTo string, encryptFor *hub.Hub) (*Exp
// Create base terminal for expansion.
base, initData, tErr := terminal.NewLocalBaseTerminal(
module.Ctx,
module.mgr.Ctx(),
0, // Ignore; The ID of the operation is used for communication.
from.FmtID(),
encryptFor,
@@ -81,7 +81,7 @@ func ExpandTo(from terminal.Terminal, routeTo string, encryptFor *hub.Hub) (*Exp
}
// Start Workers.
base.StartWorkers(module, "expansion terminal")
base.StartWorkers(module.mgr, "expansion terminal")
return expansion, nil
}

20
spn/docks/terminal_expansion_test.go
View File

@@ -1,6 +1,7 @@
package docks
import (
"context"
"fmt"
"os"
"runtime/pprof"
@@ -75,7 +76,7 @@ func testExpansion( //nolint:maintidx,thelper
serverCountTo uint64,
inParallel bool,
) {
testID += fmt.Sprintf(":encrypt=%v,flowType=%d,parallel=%v", terminalOpts.Encrypt, terminalOpts.FlowControl, inParallel)
testID += fmt.Sprintf(":encrypt=%v,flowCtrl=%d,parallel=%v", terminalOpts.Encrypt, terminalOpts.FlowControl, inParallel)
var identity2, identity3, identity4 *cabin.Identity
var connectedHub2, connectedHub3, connectedHub4 *hub.Hub
@@ -95,6 +96,9 @@ func testExpansion( //nolint:maintidx,thelper
var craneWg sync.WaitGroup
craneWg.Add(6)
craneCtx, cancelCraneCtx := context.WithCancel(context.Background())
defer cancelCraneCtx()
go func() {
var err error
crane1, err = NewCrane(ship1to2, connectedHub2, nil)
@@ -102,7 +106,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane1: %s", testID, err))
}
crane1.ID = "c1"
err = crane1.Start(module.Ctx)
err = crane1.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane1: %s", testID, err))
}
@@ -116,7 +120,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane2to1: %s", testID, err))
}
crane2to1.ID = "c2to1"
err = crane2to1.Start(module.Ctx)
err = crane2to1.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane2to1: %s", testID, err))
}
@@ -130,7 +134,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane2to3: %s", testID, err))
}
crane2to3.ID = "c2to3"
err = crane2to3.Start(module.Ctx)
err = crane2to3.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane2to3: %s", testID, err))
}
@@ -144,7 +148,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane3to2: %s", testID, err))
}
crane3to2.ID = "c3to2"
err = crane3to2.Start(module.Ctx)
err = crane3to2.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane3to2: %s", testID, err))
}
@@ -158,7 +162,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane3to4: %s", testID, err))
}
crane3to4.ID = "c3to4"
err = crane3to4.Start(module.Ctx)
err = crane3to4.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane3to4: %s", testID, err))
}
@@ -172,7 +176,7 @@ func testExpansion( //nolint:maintidx,thelper
panic(fmt.Sprintf("expansion test %s could not create crane4: %s", testID, err))
}
crane4.ID = "c4"
err = crane4.Start(module.Ctx)
err = crane4.Start(craneCtx)
if err != nil {
panic(fmt.Sprintf("expansion test %s could not start crane4: %s", testID, err))
}
@@ -288,7 +292,7 @@ func testExpansion( //nolint:maintidx,thelper
op1.Wait()
}
// Wait for completion.
// Signal completion.
close(finished)
// Wait a little so that all errors can be propagated, so we can truly see