wip: migrate to mono-repo. SPN has already been moved to spn/

spn/docks/bandwidth_test.go

@@ -0,0 +1,90 @@
+package docks
+
+import (
+	"testing"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+func TestEffectiveBandwidth(t *testing.T) { //nolint:paralleltest // Run alone.
+	// Skip in CI.
+	if testing.Short() {
+		t.Skip()
+	}
+
+	var (
+		bwTestDelay            = 50 * time.Millisecond
+		bwTestQueueSize uint32 = 1000
+		bwTestVolume           = 10000000 // 10MB
+		bwTestTime             = 10 * time.Second
+	)
+
+	// Create test terminal pair.
+	a, b, err := terminal.NewSimpleTestTerminalPair(
+		bwTestDelay,
+		int(bwTestQueueSize),
+		&terminal.TerminalOpts{
+			FlowControl:     terminal.FlowControlDFQ,
+			FlowControlSize: bwTestQueueSize,
+		},
+	)
+	if err != nil {
+		t.Fatalf("failed to create test terminal pair: %s", err)
+	}
+
+	// Grant permission for op on remote terminal and start op.
+	b.GrantPermission(terminal.IsCraneController)
+
+	// Re-use the capacity test for the bandwidth test.
+	op := &CapacityTestOp{
+		opts: &CapacityTestOptions{
+			TestVolume: bwTestVolume,
+			MaxTime:    bwTestTime,
+			testing:    true,
+		},
+		recvQueue:       make(chan *terminal.Msg),
+		dataSent:        new(int64),
+		dataSentWasAckd: abool.New(),
+		result:          make(chan *terminal.Error, 1),
+	}
+	// Disable sender again.
+	op.senderStarted = true
+	op.dataSentWasAckd.Set()
+	// Make capacity test request.
+	request, err := dsd.Dump(op.opts, dsd.CBOR)
+	if err != nil {
+		t.Fatal(terminal.ErrInternalError.With("failed to serialize capactity test options: %w", err))
+	}
+	// Send test request.
+	tErr := a.StartOperation(op, container.New(request), 1*time.Second)
+	if tErr != nil {
+		t.Fatal(tErr)
+	}
+	// Start handler.
+	module.StartWorker("op capacity handler", op.handler)
+
+	// Wait for result and check error.
+	tErr = <-op.Result()
+	if !tErr.IsOK() {
+		t.Fatalf("op failed: %s", tErr)
+	}
+	t.Logf("measured capacity: %d bit/s", op.testResult)
+
+	// Calculate expected bandwidth.
+	expectedBitsPerSecond := (float64(capacityTestMsgSize*8*int64(bwTestQueueSize)) / float64(bwTestDelay)) * float64(time.Second)
+	t.Logf("expected capacity: %f bit/s", expectedBitsPerSecond)
+
+	// Check if measured bandwidth is within parameters.
+	if float64(op.testResult) > expectedBitsPerSecond*1.6 {
+		t.Fatal("measured capacity too high")
+	}
+	// TODO: Check if we can raise this to at least 90%.
+	if float64(op.testResult) < expectedBitsPerSecond*0.2 {
+		t.Fatal("measured capacity too low")
+	}
+}

spn/docks/controller.go

@@ -0,0 +1,100 @@
+package docks
+
+import (
+	"github.com/safing/portbase/container"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// CraneControllerTerminal is a terminal for the crane itself.
+type CraneControllerTerminal struct {
+	*terminal.TerminalBase
+
+	Crane *Crane
+}
+
+// NewLocalCraneControllerTerminal returns a new local crane controller.
+func NewLocalCraneControllerTerminal(
+	crane *Crane,
+	initMsg *terminal.TerminalOpts,
+) (*CraneControllerTerminal, *container.Container, *terminal.Error) {
+	// Remove unnecessary options from the crane controller.
+	initMsg.Padding = 0
+
+	// Create Terminal Base.
+	t, initData, err := terminal.NewLocalBaseTerminal(
+		crane.ctx,
+		0,
+		crane.ID,
+		nil,
+		initMsg,
+		terminal.UpstreamSendFunc(crane.sendImportantTerminalMsg),
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return initCraneController(crane, t, initMsg), initData, nil
+}
+
+// NewRemoteCraneControllerTerminal returns a new remote crane controller.
+func NewRemoteCraneControllerTerminal(
+	crane *Crane,
+	initData *container.Container,
+) (*CraneControllerTerminal, *terminal.TerminalOpts, *terminal.Error) {
+	// Create Terminal Base.
+	t, initMsg, err := terminal.NewRemoteBaseTerminal(
+		crane.ctx,
+		0,
+		crane.ID,
+		nil,
+		initData,
+		terminal.UpstreamSendFunc(crane.sendImportantTerminalMsg),
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return initCraneController(crane, t, initMsg), initMsg, nil
+}
+
+func initCraneController(
+	crane *Crane,
+	t *terminal.TerminalBase,
+	initMsg *terminal.TerminalOpts,
+) *CraneControllerTerminal {
+	// Create Crane Terminal and assign it as the extended Terminal.
+	cct := &CraneControllerTerminal{
+		TerminalBase: t,
+		Crane:        crane,
+	}
+	t.SetTerminalExtension(cct)
+
+	// Assign controller to crane.
+	crane.Controller = cct
+	crane.terminals[cct.ID()] = cct
+
+	// Copy the options to the crane itself.
+	crane.opts = *initMsg
+
+	// Grant crane controller permission.
+	t.GrantPermission(terminal.IsCraneController)
+
+	// Start workers.
+	t.StartWorkers(module, "crane controller terminal")
+
+	return cct
+}
+
+// HandleAbandon gives the terminal the ability to cleanly shut down.
+func (controller *CraneControllerTerminal) HandleAbandon(err *terminal.Error) (errorToSend *terminal.Error) {
+	// Abandon terminal.
+	controller.Crane.AbandonTerminal(0, err)
+
+	return err
+}
+
+// HandleDestruction gives the terminal the ability to clean up.
+func (controller *CraneControllerTerminal) HandleDestruction(err *terminal.Error) {
+	// Stop controlled crane.
+	controller.Crane.Stop(nil)
+}

spn/docks/crane.go

@@ -0,0 +1,913 @@
+package docks
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/jess"
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/varint"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/spn/cabin"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/ships"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	// QOTD holds the quote of the day to return on idling unused connections.
+	QOTD = "Privacy is not an option, and it shouldn't be the price we accept for just getting on the Internet.\nGary Kovacs\n"
+
+	// maxUnloadSize defines the maximum size of a message to unload.
+	maxUnloadSize            = 16384
+	maxSegmentLength         = 16384
+	maxCraneStoppingDuration = 6 * time.Hour
+	maxCraneStopDuration     = 10 * time.Second
+)
+
+var (
+	// optimalMinLoadSize defines minimum for Crane.targetLoadSize.
+	optimalMinLoadSize = 3072 // Targeting around 4096.
+
+	// loadingMaxWaitDuration is the maximum time a crane will wait for
+	// additional data to send.
+	loadingMaxWaitDuration = 5 * time.Millisecond
+)
+
+// Errors.
+var (
+	ErrDone = errors.New("crane is done")
+)
+
+// Crane is the primary duplexer and connection manager.
+type Crane struct {
+	// ID is the ID of the Crane.
+	ID string
+	// opts holds options.
+	opts terminal.TerminalOpts
+
+	// ctx is the context of the Terminal.
+	ctx context.Context
+	// cancelCtx cancels ctx.
+	cancelCtx context.CancelFunc
+	// stopping indicates if the Crane will be stopped soon. The Crane may still
+	// be used until stopped, but must not be advertised anymore.
+	stopping *abool.AtomicBool
+	// stopped indicates if the Crane has been stopped. Whoever stopped the Crane
+	// already took care of notifying everyone, so a silent fail is normally the
+	// best response.
+	stopped *abool.AtomicBool
+	// authenticated indicates if there is has been any successful authentication.
+	authenticated *abool.AtomicBool
+
+	// ConnectedHub is the identity of the remote Hub.
+	ConnectedHub *hub.Hub
+	// NetState holds the network optimization state.
+	// It must always be set and the reference must not be changed.
+	// Access to fields within are coordinated by itself.
+	NetState *NetworkOptimizationState
+	// identity is identity of this instance and is usually only populated on a server.
+	identity *cabin.Identity
+
+	// jession is the jess session used for encryption.
+	jession *jess.Session
+	// jessionLock locks jession.
+	jessionLock sync.Mutex
+
+	// Controller is the Crane's Controller Terminal.
+	Controller *CraneControllerTerminal
+
+	// ship represents the underlying physical connection.
+	ship ships.Ship
+	// unloading moves containers from the ship to the crane.
+	unloading chan *container.Container
+	// loading moves containers from the crane to the ship.
+	loading chan *container.Container
+	// terminalMsgs holds containers from terminals waiting to be laoded.
+	terminalMsgs chan *terminal.Msg
+	// controllerMsgs holds important containers from terminals waiting to be laoded.
+	controllerMsgs chan *terminal.Msg
+
+	// terminals holds all the connected terminals.
+	terminals map[uint32]terminal.Terminal
+	// terminalsLock locks terminals.
+	terminalsLock sync.Mutex
+	// nextTerminalID holds the next terminal ID.
+	nextTerminalID uint32
+
+	// targetLoadSize defines the optimal loading size.
+	targetLoadSize int
+}
+
+// NewCrane returns a new crane.
+func NewCrane(ship ships.Ship, connectedHub *hub.Hub, id *cabin.Identity) (*Crane, error) {
+	// Cranes always run in module context.
+	ctx, cancelCtx := context.WithCancel(module.Ctx)
+
+	newCrane := &Crane{
+		ctx:           ctx,
+		cancelCtx:     cancelCtx,
+		stopping:      abool.NewBool(false),
+		stopped:       abool.NewBool(false),
+		authenticated: abool.NewBool(false),
+
+		ConnectedHub: connectedHub,
+		NetState:     newNetworkOptimizationState(),
+		identity:     id,
+
+		ship:           ship,
+		unloading:      make(chan *container.Container),
+		loading:        make(chan *container.Container, 100),
+		terminalMsgs:   make(chan *terminal.Msg, 100),
+		controllerMsgs: make(chan *terminal.Msg, 100),
+
+		terminals: make(map[uint32]terminal.Terminal),
+	}
+	err := registerCrane(newCrane)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register crane: %w", err)
+	}
+
+	// Shift next terminal IDs on the server.
+	if !ship.IsMine() {
+		newCrane.nextTerminalID += 4
+	}
+
+	// Calculate target load size.
+	loadSize := ship.LoadSize()
+	if loadSize <= 0 {
+		loadSize = ships.BaseMTU
+	}
+	newCrane.targetLoadSize = loadSize
+	for newCrane.targetLoadSize < optimalMinLoadSize {
+		newCrane.targetLoadSize += loadSize
+	}
+	// Subtract overhead needed for encryption.
+	newCrane.targetLoadSize -= 25 // Manually tested for jess.SuiteWireV1
+	// Subtract space needed for length encoding the final chunk.
+	newCrane.targetLoadSize -= varint.EncodedSize(uint64(newCrane.targetLoadSize))
+
+	return newCrane, nil
+}
+
+// IsMine returns whether the crane was started on this side.
+func (crane *Crane) IsMine() bool {
+	return crane.ship.IsMine()
+}
+
+// Public returns whether the crane has been published.
+func (crane *Crane) Public() bool {
+	return crane.ship.Public()
+}
+
+// IsStopping returns whether the crane is stopping.
+func (crane *Crane) IsStopping() bool {
+	return crane.stopping.IsSet()
+}
+
+// MarkStoppingRequested marks the crane as stopping requested.
+func (crane *Crane) MarkStoppingRequested() {
+	crane.NetState.lock.Lock()
+	defer crane.NetState.lock.Unlock()
+
+	if !crane.NetState.stoppingRequested {
+		crane.NetState.stoppingRequested = true
+		crane.startSyncStateOp()
+	}
+}
+
+// MarkStopping marks the crane as stopping.
+func (crane *Crane) MarkStopping() (stopping bool) {
+	// Can only stop owned cranes.
+	if !crane.IsMine() {
+		return false
+	}
+
+	if !crane.stopping.SetToIf(false, true) {
+		return false
+	}
+
+	crane.NetState.lock.Lock()
+	defer crane.NetState.lock.Unlock()
+	crane.NetState.markedStoppingAt = time.Now()
+
+	crane.startSyncStateOp()
+	return true
+}
+
+// AbortStopping aborts the stopping.
+func (crane *Crane) AbortStopping() (aborted bool) {
+	aborted = crane.stopping.SetToIf(true, false)
+
+	crane.NetState.lock.Lock()
+	defer crane.NetState.lock.Unlock()
+
+	abortedStoppingRequest := crane.NetState.stoppingRequested
+	crane.NetState.stoppingRequested = false
+	crane.NetState.markedStoppingAt = time.Time{}
+
+	// Sync if any state changed.
+	if aborted || abortedStoppingRequest {
+		crane.startSyncStateOp()
+	}
+
+	return aborted
+}
+
+// Authenticated returns whether the other side of the crane has authenticated
+// itself with an access code.
+func (crane *Crane) Authenticated() bool {
+	return crane.authenticated.IsSet()
+}
+
+// Publish publishes the connection as a lane.
+func (crane *Crane) Publish() error {
+	// Check if crane is connected.
+	if crane.ConnectedHub == nil {
+		return fmt.Errorf("spn/docks: %s: cannot publish without defined connected hub", crane)
+	}
+
+	// Submit metrics.
+	if !crane.Public() {
+		newPublicCranes.Inc()
+	}
+
+	// Mark crane as public.
+	maskedID := crane.ship.MaskAddress(crane.ship.RemoteAddr())
+	crane.ship.MarkPublic()
+
+	// Assign crane to make it available to others.
+	AssignCrane(crane.ConnectedHub.ID, crane)
+
+	log.Infof("spn/docks: %s (was %s) is now public", crane, maskedID)
+	return nil
+}
+
+// LocalAddr returns ship's local address.
+func (crane *Crane) LocalAddr() net.Addr {
+	return crane.ship.LocalAddr()
+}
+
+// RemoteAddr returns ship's local address.
+func (crane *Crane) RemoteAddr() net.Addr {
+	return crane.ship.RemoteAddr()
+}
+
+// Transport returns ship's transport.
+func (crane *Crane) Transport() *hub.Transport {
+	return crane.ship.Transport()
+}
+
+func (crane *Crane) getNextTerminalID() uint32 {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	for {
+		// Bump to next ID.
+		crane.nextTerminalID += 8
+
+		// Check if it's free.
+		_, ok := crane.terminals[crane.nextTerminalID]
+		if !ok {
+			return crane.nextTerminalID
+		}
+	}
+}
+
+func (crane *Crane) terminalCount() int {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	return len(crane.terminals)
+}
+
+func (crane *Crane) getTerminal(id uint32) (t terminal.Terminal, ok bool) {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	t, ok = crane.terminals[id]
+	return
+}
+
+func (crane *Crane) setTerminal(t terminal.Terminal) {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	crane.terminals[t.ID()] = t
+}
+
+func (crane *Crane) deleteTerminal(id uint32) (t terminal.Terminal, ok bool) {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	t, ok = crane.terminals[id]
+	if ok {
+		delete(crane.terminals, id)
+		return t, true
+	}
+	return nil, false
+}
+
+// AbandonTerminal abandons the terminal with the given ID.
+func (crane *Crane) AbandonTerminal(id uint32, err *terminal.Error) {
+	// Get active terminal.
+	t, ok := crane.deleteTerminal(id)
+	if ok {
+		// If the terminal was registered, abandon it.
+
+		// Log reason the terminal is ending. Override stopping error with nil.
+		switch {
+		case err == nil || err.IsOK():
+			log.Debugf("spn/docks: %T %s is being abandoned", t, t.FmtID())
+		case err.Is(terminal.ErrStopping):
+			err = nil
+			log.Debugf("spn/docks: %T %s is being abandoned by peer", t, t.FmtID())
+		case err.Is(terminal.ErrNoActivity):
+			err = nil
+			log.Debugf("spn/docks: %T %s is being abandoned due to no activity", t, t.FmtID())
+		default:
+			log.Warningf("spn/docks: %T %s: %s", t, t.FmtID(), err)
+		}
+
+		// Call the terminal's abandon function.
+		t.Abandon(err)
+	} else { //nolint:gocritic
+		// When a crane terminal is abandoned, it calls crane.AbandonTerminal when
+		// finished. This time, the terminal won't be in the registry anymore and
+		// it finished shutting down, so we can now check if the crane needs to be
+		// stopped.
+
+		// If the crane is stopping, check if we can stop.
+		// We can stop when all terminals are abandoned or after a timeout.
+		// FYI: The crane controller will always take up one slot.
+		if crane.stopping.IsSet() &&
+			crane.terminalCount() <= 1 {
+			// Stop the crane in worker, so the caller can do some work.
+			module.StartWorker("retire crane", func(_ context.Context) error {
+				// Let enough time for the last errors to be sent, as terminals are abandoned in a goroutine.
+				time.Sleep(3 * time.Second)
+				crane.Stop(nil)
+				return nil
+			})
+		}
+	}
+}
+
+func (crane *Crane) sendImportantTerminalMsg(msg *terminal.Msg, timeout time.Duration) *terminal.Error {
+	select {
+	case crane.controllerMsgs <- msg:
+		return nil
+	case <-crane.ctx.Done():
+		msg.Finish()
+		return terminal.ErrCanceled
+	}
+}
+
+// Send is used by others to send a message through the crane.
+func (crane *Crane) Send(msg *terminal.Msg, timeout time.Duration) *terminal.Error {
+	select {
+	case crane.terminalMsgs <- msg:
+		return nil
+	case <-crane.ctx.Done():
+		msg.Finish()
+		return terminal.ErrCanceled
+	}
+}
+
+func (crane *Crane) encrypt(shipment *container.Container) (encrypted *container.Container, err error) {
+	// Skip if encryption is not enabled.
+	if crane.jession == nil {
+		return shipment, nil
+	}
+
+	crane.jessionLock.Lock()
+	defer crane.jessionLock.Unlock()
+
+	letter, err := crane.jession.Close(shipment.CompileData())
+	if err != nil {
+		return nil, err
+	}
+
+	encrypted, err = letter.ToWire()
+	if err != nil {
+		return nil, fmt.Errorf("failed to pack letter: %w", err)
+	}
+
+	return encrypted, nil
+}
+
+func (crane *Crane) decrypt(shipment *container.Container) (decrypted *container.Container, err error) {
+	// Skip if encryption is not enabled.
+	if crane.jession == nil {
+		return shipment, nil
+	}
+
+	crane.jessionLock.Lock()
+	defer crane.jessionLock.Unlock()
+
+	letter, err := jess.LetterFromWire(shipment)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse letter: %w", err)
+	}
+
+	decryptedData, err := crane.jession.Open(letter)
+	if err != nil {
+		return nil, err
+	}
+
+	return container.New(decryptedData), nil
+}
+
+func (crane *Crane) unloader(workerCtx context.Context) error {
+	// Unclean shutdown safeguard.
+	defer crane.Stop(terminal.ErrUnknownError.With("unloader died"))
+
+	for {
+		// Get first couple bytes to get the packet length.
+		// 2 bytes are enough to encode 65535.
+		// On the other hand, packets can be only 2 bytes small.
+		lenBuf := make([]byte, 2)
+		err := crane.unloadUntilFull(lenBuf)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				crane.Stop(terminal.ErrStopping.With("connection closed"))
+			} else {
+				crane.Stop(terminal.ErrInternalError.With("failed to unload: %w", err))
+			}
+			return nil
+		}
+
+		// Unpack length.
+		containerLen, n, err := varint.Unpack64(lenBuf)
+		if err != nil {
+			crane.Stop(terminal.ErrMalformedData.With("failed to get container length: %w", err))
+			return nil
+		}
+		switch {
+		case containerLen <= 0:
+			crane.Stop(terminal.ErrMalformedData.With("received empty container with length %d", containerLen))
+			return nil
+		case containerLen > maxUnloadSize:
+			crane.Stop(terminal.ErrMalformedData.With("received oversized container with length %d", containerLen))
+			return nil
+		}
+
+		// Build shipment.
+		var shipmentBuf []byte
+		leftovers := len(lenBuf) - n
+
+		if leftovers == int(containerLen) {
+			// We already have all the shipment data.
+			shipmentBuf = lenBuf[n:]
+		} else {
+			// Create a shipment buffer, copy leftovers and read the rest from the connection.
+			shipmentBuf = make([]byte, containerLen)
+			if leftovers > 0 {
+				copy(shipmentBuf, lenBuf[n:])
+			}
+
+			// Read remaining shipment.
+			err = crane.unloadUntilFull(shipmentBuf[leftovers:])
+			if err != nil {
+				crane.Stop(terminal.ErrInternalError.With("failed to unload: %w", err))
+				return nil
+			}
+		}
+
+		// Submit to handler.
+		select {
+		case <-crane.ctx.Done():
+			crane.Stop(nil)
+			return nil
+		case crane.unloading <- container.New(shipmentBuf):
+		}
+	}
+}
+
+func (crane *Crane) unloadUntilFull(buf []byte) error {
+	var bytesRead int
+	for {
+		// Get shipment from ship.
+		n, err := crane.ship.UnloadTo(buf[bytesRead:])
+		if err != nil {
+			return err
+		}
+		if n == 0 {
+			log.Tracef("spn/docks: %s unloaded 0 bytes", crane)
+		}
+		bytesRead += n
+
+		// Return if buffer has been fully filled.
+		if bytesRead == len(buf) {
+			// Submit metrics.
+			crane.submitCraneTrafficStats(bytesRead)
+			crane.NetState.ReportTraffic(uint64(bytesRead), true)
+
+			return nil
+		}
+	}
+}
+
+func (crane *Crane) handler(workerCtx context.Context) error {
+	var partialShipment *container.Container
+	var segmentLength uint32
+
+	// Unclean shutdown safeguard.
+	defer crane.Stop(terminal.ErrUnknownError.With("handler died"))
+
+handling:
+	for {
+		select {
+		case <-crane.ctx.Done():
+			crane.Stop(nil)
+			return nil
+
+		case shipment := <-crane.unloading:
+			// log.Debugf("spn/crane %s: before decrypt: %v ... %v", crane.ID, c.CompileData()[:10], c.CompileData()[c.Length()-10:])
+
+			// Decrypt shipment.
+			shipment, err := crane.decrypt(shipment)
+			if err != nil {
+				crane.Stop(terminal.ErrIntegrity.With("failed to decrypt: %w", err))
+				return nil
+			}
+
+			// Process all segments/containers of the shipment.
+			for shipment.HoldsData() {
+				if partialShipment != nil {
+					// Continue processing partial segment.
+					// Append new shipment to previous partial segment.
+					partialShipment.AppendContainer(shipment)
+					shipment, partialShipment = partialShipment, nil
+				}
+
+				// Get next segment length.
+				if segmentLength == 0 {
+					segmentLength, err = shipment.GetNextN32()
+					if err != nil {
+						if errors.Is(err, varint.ErrBufTooSmall) {
+							// Continue handling when there is not yet enough data.
+							partialShipment = shipment
+							segmentLength = 0
+							continue handling
+						}
+
+						crane.Stop(terminal.ErrMalformedData.With("failed to get segment length: %w", err))
+						return nil
+					}
+
+					if segmentLength == 0 {
+						// Remainder is padding.
+						continue handling
+					}
+
+					// Check if the segment is within the boundary.
+					if segmentLength > maxSegmentLength {
+						crane.Stop(terminal.ErrMalformedData.With("received oversized segment with length %d", segmentLength))
+						return nil
+					}
+				}
+
+				// Check if we have enough data for the segment.
+				if uint32(shipment.Length()) < segmentLength {
+					partialShipment = shipment
+					continue handling
+				}
+
+				// Get segment from shipment.
+				segment, err := shipment.GetAsContainer(int(segmentLength))
+				if err != nil {
+					crane.Stop(terminal.ErrMalformedData.With("failed to get segment: %w", err))
+					return nil
+				}
+				segmentLength = 0
+
+				// Get terminal ID and message type of segment.
+				terminalID, terminalMsgType, err := terminal.ParseIDType(segment)
+				if err != nil {
+					crane.Stop(terminal.ErrMalformedData.With("failed to get terminal ID and msg type: %w", err))
+					return nil
+				}
+
+				switch terminalMsgType {
+				case terminal.MsgTypeInit:
+					crane.establishTerminal(terminalID, segment)
+
+				case terminal.MsgTypeData, terminal.MsgTypePriorityData:
+					// Get terminal and let it further handle the message.
+					t, ok := crane.getTerminal(terminalID)
+					if ok {
+						// Create msg and set priority.
+						msg := terminal.NewEmptyMsg()
+						msg.FlowID = terminalID
+						msg.Type = terminalMsgType
+						msg.Data = segment
+						if msg.Type == terminal.MsgTypePriorityData {
+							msg.Unit.MakeHighPriority()
+						}
+						// Deliver to terminal.
+						deliveryErr := t.Deliver(msg)
+						if deliveryErr != nil {
+							msg.Finish()
+							// This is a hot path. Start a worker for abandoning the terminal.
+							module.StartWorker("end terminal", func(_ context.Context) error {
+								crane.AbandonTerminal(t.ID(), deliveryErr.Wrap("failed to deliver data"))
+								return nil
+							})
+						}
+					} else {
+						log.Tracef("spn/docks: %s received msg for unknown terminal %d", crane, terminalID)
+					}
+
+				case terminal.MsgTypeStop:
+					// Parse error.
+					receivedErr, err := terminal.ParseExternalError(segment.CompileData())
+					if err != nil {
+						log.Warningf("spn/docks: %s failed to parse abandon error: %s", crane, err)
+						receivedErr = terminal.ErrUnknownError.AsExternal()
+					}
+					// This is a hot path. Start a worker for abandoning the terminal.
+					module.StartWorker("end terminal", func(_ context.Context) error {
+						crane.AbandonTerminal(terminalID, receivedErr)
+						return nil
+					})
+				}
+			}
+		}
+	}
+}
+
+func (crane *Crane) loader(workerCtx context.Context) (err error) {
+	shipment := container.New()
+	var partialShipment *container.Container
+	var loadingTimer *time.Timer
+
+	// Unclean shutdown safeguard.
+	defer crane.Stop(terminal.ErrUnknownError.With("loader died"))
+
+	// Return the loading wait channel if waiting.
+	loadNow := func() <-chan time.Time {
+		if loadingTimer != nil {
+			return loadingTimer.C
+		}
+		return nil
+	}
+
+	// Make sure any received message is finished
+	var msg, firstMsg *terminal.Msg
+	defer msg.Finish()
+	defer firstMsg.Finish()
+
+	for {
+		// Reset first message in shipment.
+		firstMsg.Finish()
+		firstMsg = nil
+
+	fillingShipment:
+		for shipment.Length() < crane.targetLoadSize {
+			// Gather segments until shipment is filled.
+
+			// Prioritize messages from the controller.
+			select {
+			case msg = <-crane.controllerMsgs:
+			case <-crane.ctx.Done():
+				crane.Stop(nil)
+				return nil
+
+			default:
+				// Then listen for all.
+				select {
+				case msg = <-crane.controllerMsgs:
+				case msg = <-crane.terminalMsgs:
+				case <-loadNow():
+					break fillingShipment
+				case <-crane.ctx.Done():
+					crane.Stop(nil)
+					return nil
+				}
+			}
+
+			// Debug unit leaks.
+			msg.Debug()
+
+			// Handle new message.
+			if msg != nil {
+				// Pack msg and add to segment.
+				msg.Pack()
+				newSegment := msg.Data
+
+				// Check if this is the first message.
+				// This is the only message where we wait for a slot.
+				if firstMsg == nil {
+					firstMsg = msg
+					firstMsg.Unit.WaitForSlot()
+				} else {
+					msg.Finish()
+				}
+
+				// Check length.
+				if newSegment.Length() > maxSegmentLength {
+					log.Warningf("spn/docks: %s ignored oversized segment with length %d", crane, newSegment.Length())
+					continue fillingShipment
+				}
+
+				// Append to shipment.
+				shipment.AppendContainer(newSegment)
+
+				// Set loading max wait timer on first segment.
+				if loadingTimer == nil {
+					loadingTimer = time.NewTimer(loadingMaxWaitDuration)
+				}
+
+			} else if crane.stopped.IsSet() {
+				// If there is no new segment, this might have been triggered by a
+				// closed channel. Check if the crane is still active.
+				return nil
+			}
+		}
+
+	sendingShipment:
+		for {
+			// Check if we are over the target load size and split the shipment.
+			if shipment.Length() > crane.targetLoadSize {
+				partialShipment, err = shipment.GetAsContainer(crane.targetLoadSize)
+				if err != nil {
+					crane.Stop(terminal.ErrInternalError.With("failed to split segment: %w", err))
+					return nil
+				}
+				shipment, partialShipment = partialShipment, shipment
+			}
+
+			// Load shipment.
+			err = crane.load(shipment)
+			if err != nil {
+				crane.Stop(terminal.ErrShipSunk.With("failed to load shipment: %w", err))
+				return nil
+			}
+
+			// Reset loading timer.
+			loadingTimer = nil
+
+			// Continue loading with partial shipment, or a new one.
+			if partialShipment != nil {
+				// Continue loading with a partial previous shipment.
+				shipment, partialShipment = partialShipment, nil
+
+				// If shipment is not big enough to send immediately, wait for more data.
+				if shipment.Length() < crane.targetLoadSize {
+					loadingTimer = time.NewTimer(loadingMaxWaitDuration)
+					break sendingShipment
+				}
+
+			} else {
+				// Continue loading with new shipment.
+				shipment = container.New()
+				break sendingShipment
+			}
+		}
+	}
+}
+
+func (crane *Crane) load(c *container.Container) error {
+	// Add Padding if needed.
+	if crane.opts.Padding > 0 {
+		paddingNeeded := int(crane.opts.Padding) -
+			((c.Length() + varint.EncodedSize(uint64(c.Length()))) % int(crane.opts.Padding))
+		// As the length changes slightly with the padding, we should avoid loading
+		// lengths around the varint size hops:
+		// - 128
+		// - 16384
+		// - 2097152
+		// - 268435456
+
+		// Pad to target load size at maximum.
+		maxPadding := crane.targetLoadSize - c.Length()
+		if paddingNeeded > maxPadding {
+			paddingNeeded = maxPadding
+		}
+
+		if paddingNeeded > 0 {
+			// Add padding indicator.
+			c.Append([]byte{0})
+			paddingNeeded--
+
+			// Add needed padding data.
+			if paddingNeeded > 0 {
+				padding, err := rng.Bytes(paddingNeeded)
+				if err != nil {
+					log.Debugf("spn/docks: %s failed to get random padding data, using zeros instead", crane)
+					padding = make([]byte, paddingNeeded)
+				}
+				c.Append(padding)
+			}
+		}
+	}
+
+	// Encrypt shipment.
+	c, err := crane.encrypt(c)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt: %w", err)
+	}
+
+	// Finalize data.
+	c.PrependLength()
+	readyToSend := c.CompileData()
+
+	// Submit metrics.
+	crane.submitCraneTrafficStats(len(readyToSend))
+	crane.NetState.ReportTraffic(uint64(len(readyToSend)), false)
+
+	// Load onto ship.
+	err = crane.ship.Load(readyToSend)
+	if err != nil {
+		return fmt.Errorf("failed to load ship: %w", err)
+	}
+
+	return nil
+}
+
+// Stop stops the crane.
+func (crane *Crane) Stop(err *terminal.Error) {
+	if !crane.stopped.SetToIf(false, true) {
+		return
+	}
+
+	// Log error message.
+	if err != nil {
+		if err.IsOK() {
+			log.Infof("spn/docks: %s is done", crane)
+		} else {
+			log.Warningf("spn/docks: %s is stopping: %s", crane, err)
+		}
+	}
+
+	// Unregister crane.
+	unregisterCrane(crane)
+
+	// Stop all terminals.
+	for _, t := range crane.allTerms() {
+		t.Abandon(err) // Async!
+	}
+
+	// Stop controller.
+	if crane.Controller != nil {
+		crane.Controller.Abandon(err) // Async!
+	}
+
+	// Wait shortly for all terminals to finish abandoning.
+	waitStep := 50 * time.Millisecond
+	for i := time.Duration(0); i < maxCraneStopDuration; i += waitStep {
+		// Check if all terminals are done.
+		if crane.terminalCount() == 0 {
+			break
+		}
+
+		time.Sleep(waitStep)
+	}
+
+	// Close connection.
+	crane.ship.Sink()
+
+	// Cancel crane context.
+	crane.cancelCtx()
+
+	// Notify about change.
+	crane.NotifyUpdate()
+}
+
+func (crane *Crane) allTerms() []terminal.Terminal {
+	crane.terminalsLock.Lock()
+	defer crane.terminalsLock.Unlock()
+
+	terms := make([]terminal.Terminal, 0, len(crane.terminals))
+	for _, term := range crane.terminals {
+		terms = append(terms, term)
+	}
+
+	return terms
+}
+
+func (crane *Crane) String() string {
+	remoteAddr := crane.ship.RemoteAddr()
+	switch {
+	case remoteAddr == nil:
+		return fmt.Sprintf("crane %s", crane.ID)
+	case crane.ship.IsMine():
+		return fmt.Sprintf("crane %s to %s", crane.ID, crane.ship.MaskAddress(crane.ship.RemoteAddr()))
+	default:
+		return fmt.Sprintf("crane %s from %s", crane.ID, crane.ship.MaskAddress(crane.ship.RemoteAddr()))
+	}
+}
+
+// Stopped returns whether the crane has stopped.
+func (crane *Crane) Stopped() bool {
+	return crane.stopped.IsSet()
+}

spn/docks/crane_establish.go

@@ -0,0 +1,81 @@
+package docks
+
+import (
+	"context"
+	"time"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	defaultTerminalIdleTimeout = 15 * time.Minute
+	remoteTerminalIdleTimeout  = 30 * time.Minute
+)
+
+// EstablishNewTerminal establishes a new terminal with the crane.
+func (crane *Crane) EstablishNewTerminal(
+	localTerm terminal.Terminal,
+	initData *container.Container,
+) *terminal.Error {
+	// Create message.
+	msg := terminal.NewEmptyMsg()
+	msg.FlowID = localTerm.ID()
+	msg.Type = terminal.MsgTypeInit
+	msg.Data = initData
+
+	// Register terminal with crane.
+	crane.setTerminal(localTerm)
+
+	// Send message.
+	select {
+	case crane.controllerMsgs <- msg:
+		log.Debugf("spn/docks: %s initiated new terminal %d", crane, localTerm.ID())
+		return nil
+	case <-crane.ctx.Done():
+		crane.AbandonTerminal(localTerm.ID(), terminal.ErrStopping.With("initiation aborted"))
+		return terminal.ErrStopping
+	}
+}
+
+func (crane *Crane) establishTerminal(id uint32, initData *container.Container) {
+	// Create new remote crane terminal.
+	newTerminal, _, err := NewRemoteCraneTerminal(
+		crane,
+		id,
+		initData,
+	)
+	if err == nil {
+		// Connections via public cranes have a timeout.
+		if crane.Public() {
+			newTerminal.TerminalBase.SetTimeout(remoteTerminalIdleTimeout)
+		}
+		// Register terminal with crane.
+		crane.setTerminal(newTerminal)
+		log.Debugf("spn/docks: %s established new crane terminal %d", crane, newTerminal.ID())
+		return
+	}
+
+	// If something goes wrong, send an error back.
+	log.Warningf("spn/docks: %s failed to establish crane terminal: %s", crane, err)
+
+	// Build abandon message.
+	msg := terminal.NewMsg(err.Pack())
+	msg.FlowID = id
+	msg.Type = terminal.MsgTypeStop
+
+	// Send message directly, or async.
+	select {
+	case crane.terminalMsgs <- msg:
+	default:
+		// Send error async.
+		module.StartWorker("abandon terminal", func(ctx context.Context) error {
+			select {
+			case crane.terminalMsgs <- msg:
+			case <-ctx.Done():
+			}
+			return nil
+		})
+	}
+}

spn/docks/crane_init.go

@@ -0,0 +1,339 @@
+package docks
+
+import (
+	"context"
+	"time"
+
+	"github.com/safing/jess"
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/formats/varint"
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/spn/conf"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+/*
+
+Crane Init Message Format:
+used by init procedures
+
+- Data [bytes block]
+	- MsgType [varint]
+	- Data [bytes; only when MsgType is Verify or Start*]
+
+Crane Init Response Format:
+
+- Data [bytes block]
+
+Crane Operational Message Format:
+
+- Data [bytes block]
+	- possibly encrypted
+
+*/
+
+// Crane Msg Types.
+const (
+	CraneMsgTypeEnd              = 0
+	CraneMsgTypeInfo             = 1
+	CraneMsgTypeRequestHubInfo   = 2
+	CraneMsgTypeVerify           = 3
+	CraneMsgTypeStartEncrypted   = 4
+	CraneMsgTypeStartUnencrypted = 5
+)
+
+// Start starts the crane.
+func (crane *Crane) Start(callerCtx context.Context) error {
+	log.Infof("spn/docks: %s is starting", crane)
+
+	// Submit metrics.
+	newCranes.Inc()
+
+	// Start crane depending on situation.
+	var tErr *terminal.Error
+	if crane.ship.IsMine() {
+		tErr = crane.startLocal(callerCtx)
+	} else {
+		tErr = crane.startRemote(callerCtx)
+	}
+
+	// Stop crane again if starting failed.
+	if tErr != nil {
+		crane.Stop(tErr)
+		return tErr
+	}
+
+	log.Debugf("spn/docks: %s started", crane)
+	// Return an explicit nil for working "!= nil" checks.
+	return nil
+}
+
+func (crane *Crane) startLocal(callerCtx context.Context) *terminal.Error {
+	module.StartWorker("crane unloader", crane.unloader)
+
+	if !crane.ship.IsSecure() {
+		// Start encrypted channel.
+		// Check if we have all the data we need from the Hub.
+		if crane.ConnectedHub == nil {
+			return terminal.ErrIncorrectUsage.With("cannot start encrypted channel without connected hub")
+		}
+
+		// Always request hub info, as we don't know if the hub has restarted in
+		// the meantime and lost ephemeral keys.
+		hubInfoRequest := container.New(
+			varint.Pack8(CraneMsgTypeRequestHubInfo),
+		)
+		hubInfoRequest.PrependLength()
+		err := crane.ship.Load(hubInfoRequest.CompileData())
+		if err != nil {
+			return terminal.ErrShipSunk.With("failed to request hub info: %w", err)
+		}
+
+		// Wait for reply.
+		var reply *container.Container
+		select {
+		case reply = <-crane.unloading:
+		case <-time.After(30 * time.Second):
+			return terminal.ErrTimeout.With("waiting for hub info")
+		case <-crane.ctx.Done():
+			return terminal.ErrShipSunk.With("waiting for hub info")
+		case <-callerCtx.Done():
+			return terminal.ErrCanceled.With("waiting for hub info")
+		}
+
+		// Parse and import Announcement and Status.
+		announcementData, err := reply.GetNextBlock()
+		if err != nil {
+			return terminal.ErrMalformedData.With("failed to get announcement: %w", err)
+		}
+		statusData, err := reply.GetNextBlock()
+		if err != nil {
+			return terminal.ErrMalformedData.With("failed to get status: %w", err)
+		}
+		h, _, tErr := ImportAndVerifyHubInfo(
+			callerCtx,
+			crane.ConnectedHub.ID,
+			announcementData, statusData, conf.MainMapName, conf.MainMapScope,
+		)
+		if tErr != nil {
+			return tErr.Wrap("failed to import and verify hub")
+		}
+		// Update reference in case it was changed by the import.
+		crane.ConnectedHub = h
+
+		// Now, try to select a public key again.
+		signet := crane.ConnectedHub.SelectSignet()
+		if signet == nil {
+			return terminal.ErrHubNotReady.With("failed to select signet (after updating hub info)")
+		}
+
+		// Configure encryption.
+		env := jess.NewUnconfiguredEnvelope()
+		env.SuiteID = jess.SuiteWireV1
+		env.Recipients = []*jess.Signet{signet}
+
+		// Do not encrypt directly, rather get session for future use, then encrypt.
+		crane.jession, err = env.WireCorrespondence(nil)
+		if err != nil {
+			return terminal.ErrInternalError.With("failed to create encryption session: %w", err)
+		}
+	}
+
+	// Create crane controller.
+	_, initData, tErr := NewLocalCraneControllerTerminal(crane, terminal.DefaultCraneControllerOpts())
+	if tErr != nil {
+		return tErr.Wrap("failed to set up controller")
+	}
+
+	// Prepare init message for sending.
+	if crane.ship.IsSecure() {
+		initData.PrependNumber(CraneMsgTypeStartUnencrypted)
+	} else {
+		// Encrypt controller initializer.
+		letter, err := crane.jession.Close(initData.CompileData())
+		if err != nil {
+			return terminal.ErrInternalError.With("failed to encrypt initial packet: %w", err)
+		}
+		initData, err = letter.ToWire()
+		if err != nil {
+			return terminal.ErrInternalError.With("failed to pack initial packet: %w", err)
+		}
+		initData.PrependNumber(CraneMsgTypeStartEncrypted)
+	}
+
+	// Send start message.
+	initData.PrependLength()
+	err := crane.ship.Load(initData.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send init msg: %w", err)
+	}
+
+	// Start remaining workers.
+	module.StartWorker("crane loader", crane.loader)
+	module.StartWorker("crane handler", crane.handler)
+
+	return nil
+}
+
+func (crane *Crane) startRemote(callerCtx context.Context) *terminal.Error {
+	var initMsg *container.Container
+
+	module.StartWorker("crane unloader", crane.unloader)
+
+handling:
+	for {
+		// Wait for request.
+		var request *container.Container
+		select {
+		case request = <-crane.unloading:
+
+		case <-time.After(30 * time.Second):
+			return terminal.ErrTimeout.With("waiting for crane init msg")
+		case <-crane.ctx.Done():
+			return terminal.ErrShipSunk.With("waiting for crane init msg")
+		case <-callerCtx.Done():
+			return terminal.ErrCanceled.With("waiting for crane init msg")
+		}
+
+		msgType, err := request.GetNextN8()
+		if err != nil {
+			return terminal.ErrMalformedData.With("failed to parse crane msg type: %s", err)
+		}
+
+		switch msgType {
+		case CraneMsgTypeEnd:
+			// End connection.
+			return terminal.ErrStopping
+
+		case CraneMsgTypeInfo:
+			// Info is a terminating request.
+			err := crane.handleCraneInfo()
+			if err != nil {
+				return err
+			}
+			log.Debugf("spn/docks: %s sent version info", crane)
+
+		case CraneMsgTypeRequestHubInfo:
+			// Handle Hub info request.
+			err := crane.handleCraneHubInfo()
+			if err != nil {
+				return err
+			}
+			log.Debugf("spn/docks: %s sent hub info", crane)
+
+		case CraneMsgTypeVerify:
+			// Verify is a terminating request.
+			err := crane.handleCraneVerification(request)
+			if err != nil {
+				return err
+			}
+			log.Infof("spn/docks: %s sent hub verification", crane)
+
+		case CraneMsgTypeStartUnencrypted:
+			initMsg = request
+
+			// Start crane with initMsg.
+			log.Debugf("spn/docks: %s initiated unencrypted channel", crane)
+			break handling
+
+		case CraneMsgTypeStartEncrypted:
+			if crane.identity == nil {
+				return terminal.ErrIncorrectUsage.With("cannot start incoming crane without designated identity")
+			}
+
+			// Set up encryption.
+			letter, err := jess.LetterFromWire(container.New(request.CompileData()))
+			if err != nil {
+				return terminal.ErrMalformedData.With("failed to unpack initial packet: %w", err)
+			}
+			crane.jession, err = letter.WireCorrespondence(crane.identity)
+			if err != nil {
+				return terminal.ErrInternalError.With("failed to create encryption session: %w", err)
+			}
+			initMsgData, err := crane.jession.Open(letter)
+			if err != nil {
+				return terminal.ErrIntegrity.With("failed to decrypt initial packet: %w", err)
+			}
+			initMsg = container.New(initMsgData)
+
+			// Start crane with initMsg.
+			log.Debugf("spn/docks: %s initiated encrypted channel", crane)
+			break handling
+		}
+	}
+
+	_, _, err := NewRemoteCraneControllerTerminal(crane, initMsg)
+	if err != nil {
+		return err.Wrap("failed to start crane controller")
+	}
+
+	// Start remaining workers.
+	module.StartWorker("crane loader", crane.loader)
+	module.StartWorker("crane handler", crane.handler)
+
+	return nil
+}
+
+func (crane *Crane) endInit() *terminal.Error {
+	endMsg := container.New(
+		varint.Pack8(CraneMsgTypeEnd),
+	)
+	endMsg.PrependLength()
+	err := crane.ship.Load(endMsg.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send end msg: %w", err)
+	}
+	return nil
+}
+
+func (crane *Crane) handleCraneInfo() *terminal.Error {
+	// Pack info data.
+	infoData, err := dsd.Dump(info.GetInfo(), dsd.JSON)
+	if err != nil {
+		return terminal.ErrInternalError.With("failed to pack info: %w", err)
+	}
+	msg := container.New(infoData)
+
+	// Manually send reply.
+	msg.PrependLength()
+	err = crane.ship.Load(msg.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send info reply: %w", err)
+	}
+
+	return nil
+}
+
+func (crane *Crane) handleCraneHubInfo() *terminal.Error {
+	msg := container.New()
+
+	// Check if we have an identity.
+	if crane.identity == nil {
+		return terminal.ErrIncorrectUsage.With("cannot handle hub info request without designated identity")
+	}
+
+	// Add Hub Announcement.
+	announcementData, err := crane.identity.ExportAnnouncement()
+	if err != nil {
+		return terminal.ErrInternalError.With("failed to export announcement: %w", err)
+	}
+	msg.AppendAsBlock(announcementData)
+
+	// Add Hub Status.
+	statusData, err := crane.identity.ExportStatus()
+	if err != nil {
+		return terminal.ErrInternalError.With("failed to export status: %w", err)
+	}
+	msg.AppendAsBlock(statusData)
+
+	// Manually send reply.
+	msg.PrependLength()
+	err = crane.ship.Load(msg.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send hub info reply: %w", err)
+	}
+
+	return nil
+}

spn/docks/crane_netstate.go

@@ -0,0 +1,131 @@
+package docks
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// NetStatePeriodInterval defines the interval some of the net state should be reset.
+const NetStatePeriodInterval = 15 * time.Minute
+
+// NetworkOptimizationState holds data for optimization purposes.
+type NetworkOptimizationState struct {
+	lock sync.Mutex
+
+	// lastSuggestedAt holds the time when the connection to the connected Hub was last suggested by the network optimization.
+	lastSuggestedAt time.Time
+
+	// stoppingRequested signifies whether stopping this lane is requested.
+	stoppingRequested bool
+	// stoppingRequestedByPeer signifies whether stopping this lane is requested by the peer.
+	stoppingRequestedByPeer bool
+	// markedStoppingAt holds the time when the crane was last marked as stopping.
+	markedStoppingAt time.Time
+
+	lifetimeBytesIn  *uint64
+	lifetimeBytesOut *uint64
+	lifetimeStarted  time.Time
+	periodBytesIn    *uint64
+	periodBytesOut   *uint64
+	periodStarted    time.Time
+}
+
+func newNetworkOptimizationState() *NetworkOptimizationState {
+	return &NetworkOptimizationState{
+		lifetimeBytesIn:  new(uint64),
+		lifetimeBytesOut: new(uint64),
+		lifetimeStarted:  time.Now(),
+		periodBytesIn:    new(uint64),
+		periodBytesOut:   new(uint64),
+		periodStarted:    time.Now(),
+	}
+}
+
+// UpdateLastSuggestedAt sets when the lane was last suggested to the current time.
+func (netState *NetworkOptimizationState) UpdateLastSuggestedAt() {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	netState.lastSuggestedAt = time.Now()
+}
+
+// StoppingState returns when the stopping state.
+func (netState *NetworkOptimizationState) StoppingState() (requested, requestedByPeer bool, markedAt time.Time) {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	return netState.stoppingRequested, netState.stoppingRequestedByPeer, netState.markedStoppingAt
+}
+
+// RequestStoppingSuggested returns whether the crane should request stopping.
+func (netState *NetworkOptimizationState) RequestStoppingSuggested(maxNotSuggestedDuration time.Duration) bool {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	return time.Now().Add(-maxNotSuggestedDuration).After(netState.lastSuggestedAt)
+}
+
+// StoppingSuggested returns whether the crane should be marked as stopping.
+func (netState *NetworkOptimizationState) StoppingSuggested() bool {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	return netState.stoppingRequested &&
+		netState.stoppingRequestedByPeer
+}
+
+// StopSuggested returns whether the crane should be stopped.
+func (netState *NetworkOptimizationState) StopSuggested() bool {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	return netState.stoppingRequested &&
+		netState.stoppingRequestedByPeer &&
+		!netState.markedStoppingAt.IsZero() &&
+		time.Now().Add(-maxCraneStoppingDuration).After(netState.markedStoppingAt)
+}
+
+// ReportTraffic adds the reported transferred data to the traffic stats.
+func (netState *NetworkOptimizationState) ReportTraffic(bytes uint64, in bool) {
+	if in {
+		atomic.AddUint64(netState.lifetimeBytesIn, bytes)
+		atomic.AddUint64(netState.periodBytesIn, bytes)
+	} else {
+		atomic.AddUint64(netState.lifetimeBytesOut, bytes)
+		atomic.AddUint64(netState.periodBytesOut, bytes)
+	}
+}
+
+// LapsePeriod lapses the net state period, if needed.
+func (netState *NetworkOptimizationState) LapsePeriod() {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	// Reset period if interval elapsed.
+	if time.Now().Add(-NetStatePeriodInterval).After(netState.periodStarted) {
+		atomic.StoreUint64(netState.periodBytesIn, 0)
+		atomic.StoreUint64(netState.periodBytesOut, 0)
+		netState.periodStarted = time.Now()
+	}
+}
+
+// GetTrafficStats returns the traffic stats.
+func (netState *NetworkOptimizationState) GetTrafficStats() (
+	lifetimeBytesIn uint64,
+	lifetimeBytesOut uint64,
+	lifetimeStarted time.Time,
+	periodBytesIn uint64,
+	periodBytesOut uint64,
+	periodStarted time.Time,
+) {
+	netState.lock.Lock()
+	defer netState.lock.Unlock()
+
+	return atomic.LoadUint64(netState.lifetimeBytesIn),
+		atomic.LoadUint64(netState.lifetimeBytesOut),
+		netState.lifetimeStarted,
+		atomic.LoadUint64(netState.periodBytesIn),
+		atomic.LoadUint64(netState.periodBytesOut),
+		netState.periodStarted
+}

spn/docks/crane_terminal.go

@@ -0,0 +1,122 @@
+package docks
+
+import (
+	"net"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// CraneTerminal is a terminal started by a crane.
+type CraneTerminal struct {
+	*terminal.TerminalBase
+
+	// Add-Ons
+	terminal.SessionAddOn
+
+	crane *Crane
+}
+
+// NewLocalCraneTerminal returns a new local crane terminal.
+func NewLocalCraneTerminal(
+	crane *Crane,
+	remoteHub *hub.Hub,
+	initMsg *terminal.TerminalOpts,
+) (*CraneTerminal, *container.Container, *terminal.Error) {
+	// Create Terminal Base.
+	t, initData, err := terminal.NewLocalBaseTerminal(
+		crane.ctx,
+		crane.getNextTerminalID(),
+		crane.ID,
+		remoteHub,
+		initMsg,
+		crane,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return initCraneTerminal(crane, t), initData, nil
+}
+
+// NewRemoteCraneTerminal returns a new remote crane terminal.
+func NewRemoteCraneTerminal(
+	crane *Crane,
+	id uint32,
+	initData *container.Container,
+) (*CraneTerminal, *terminal.TerminalOpts, *terminal.Error) {
+	// Create Terminal Base.
+	t, initMsg, err := terminal.NewRemoteBaseTerminal(
+		crane.ctx,
+		id,
+		crane.ID,
+		crane.identity,
+		initData,
+		crane,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return initCraneTerminal(crane, t), initMsg, nil
+}
+
+func initCraneTerminal(
+	crane *Crane,
+	t *terminal.TerminalBase,
+) *CraneTerminal {
+	// Create Crane Terminal and assign it as the extended Terminal.
+	ct := &CraneTerminal{
+		TerminalBase: t,
+		crane:        crane,
+	}
+	t.SetTerminalExtension(ct)
+
+	// Start workers.
+	t.StartWorkers(module, "crane terminal")
+
+	return ct
+}
+
+// GrantPermission grants the given permissions.
+// Additionally, it will mark the crane as authenticated, if not public.
+func (t *CraneTerminal) GrantPermission(grant terminal.Permission) {
+	// Forward granted permission to base terminal.
+	t.TerminalBase.GrantPermission(grant)
+
+	// Mark crane as authenticated if not public or already authenticated.
+	if !t.crane.Public() && !t.crane.Authenticated() {
+		t.crane.authenticated.Set()
+
+		// Submit metrics.
+		newAuthenticatedCranes.Inc()
+	}
+}
+
+// LocalAddr returns the crane's local address.
+func (t *CraneTerminal) LocalAddr() net.Addr {
+	return t.crane.LocalAddr()
+}
+
+// RemoteAddr returns the crane's remote address.
+func (t *CraneTerminal) RemoteAddr() net.Addr {
+	return t.crane.RemoteAddr()
+}
+
+// Transport returns the crane's transport.
+func (t *CraneTerminal) Transport() *hub.Transport {
+	return t.crane.Transport()
+}
+
+// IsBeingAbandoned returns whether the terminal is being abandoned.
+func (t *CraneTerminal) IsBeingAbandoned() bool {
+	return t.Abandoning.IsSet()
+}
+
+// HandleDestruction gives the terminal the ability to clean up.
+// The terminal has already fully shut down at this point.
+// Should never be called directly. Call Abandon() instead.
+func (t *CraneTerminal) HandleDestruction(err *terminal.Error) {
+	t.crane.AbandonTerminal(t.ID(), err)
+}

spn/docks/crane_test.go

@@ -0,0 +1,267 @@
+package docks
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime/pprof"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/safing/portmaster/spn/cabin"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/ships"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+func TestCraneCommunication(t *testing.T) {
+	t.Parallel()
+
+	testCraneWithCounter(t, "plain-counter-load-100", false, 100, 1000)
+	testCraneWithCounter(t, "plain-counter-load-1000", false, 1000, 1000)
+	testCraneWithCounter(t, "plain-counter-load-10000", false, 10000, 1000)
+	testCraneWithCounter(t, "encrypted-counter", true, 1000, 1000)
+}
+
+func testCraneWithCounter(t *testing.T, testID string, encrypting bool, loadSize int, countTo uint64) { //nolint:unparam,thelper
+	var identity *cabin.Identity
+	var connectedHub *hub.Hub
+	if encrypting {
+		identity, connectedHub = getTestIdentity(t)
+	}
+
+	// Build ship and cranes.
+	optimalMinLoadSize = loadSize * 2
+	ship := ships.NewTestShip(!encrypting, loadSize)
+
+	var crane1, crane2 *Crane
+	var craneWg sync.WaitGroup
+	craneWg.Add(2)
+
+	go func() {
+		var err error
+		crane1, err = NewCrane(ship, connectedHub, nil)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not create crane1: %s", testID, err))
+		}
+		err = crane1.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not start crane1: %s", testID, err))
+		}
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane2, err = NewCrane(ship.Reverse(), nil, identity)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not create crane2: %s", testID, err))
+		}
+		err = crane2.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not start crane2: %s", testID, err))
+		}
+		craneWg.Done()
+	}()
+
+	craneWg.Wait()
+	t.Logf("crane test %s setup complete", testID)
+
+	// Wait async for test to complete, print stack after timeout.
+	finished := make(chan struct{})
+	go func() {
+		select {
+		case <-finished:
+		case <-time.After(10 * time.Second):
+			t.Logf("crane test %s is taking too long, print stack:", testID)
+			_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+			os.Exit(1)
+		}
+	}()
+
+	t.Logf("crane1 controller: %+v", crane1.Controller)
+	t.Logf("crane2 controller: %+v", crane2.Controller)
+
+	// Start counters for testing.
+	op1, tErr := terminal.NewCounterOp(crane1.Controller, terminal.CounterOpts{
+		ClientCountTo: countTo,
+		ServerCountTo: countTo,
+	})
+	if tErr != nil {
+		t.Fatalf("crane test %s failed to run counter op: %s", testID, tErr)
+	}
+
+	// Wait for completion.
+	op1.Wait()
+	close(finished)
+
+	// Wait a little so that all errors can be propagated, so we can truly see
+	// if we succeeded.
+	time.Sleep(1 * time.Second)
+
+	// Check errors.
+	if op1.Error != nil {
+		t.Fatalf("crane test %s counter op1 failed: %s", testID, op1.Error)
+	}
+}
+
+type StreamingTerminal struct {
+	terminal.BareTerminal
+
+	test     *testing.T
+	id       uint32
+	crane    *Crane
+	recv     chan *terminal.Msg
+	testData []byte
+}
+
+func (t *StreamingTerminal) ID() uint32 {
+	return t.id
+}
+
+func (t *StreamingTerminal) Ctx() context.Context {
+	return module.Ctx
+}
+
+func (t *StreamingTerminal) Deliver(msg *terminal.Msg) *terminal.Error {
+	t.recv <- msg
+	msg.Finish()
+	return nil
+}
+
+func (t *StreamingTerminal) Abandon(err *terminal.Error) {
+	t.crane.AbandonTerminal(t.ID(), err)
+	if err != nil {
+		t.test.Errorf("streaming terminal %d failed: %s", t.id, err)
+	}
+}
+
+func (t *StreamingTerminal) FmtID() string {
+	return fmt.Sprintf("test-%d", t.id)
+}
+
+func TestCraneLoadingUnloading(t *testing.T) {
+	t.Parallel()
+
+	testCraneWithStreaming(t, "plain-streaming", false, 100)
+	testCraneWithStreaming(t, "encrypted-streaming", true, 100)
+}
+
+func testCraneWithStreaming(t *testing.T, testID string, encrypting bool, loadSize int) { //nolint:thelper
+	var identity *cabin.Identity
+	var connectedHub *hub.Hub
+	if encrypting {
+		identity, connectedHub = getTestIdentity(t)
+	}
+
+	// Build ship and cranes.
+	optimalMinLoadSize = loadSize * 2
+	ship := ships.NewTestShip(!encrypting, loadSize)
+
+	var crane1, crane2 *Crane
+	var craneWg sync.WaitGroup
+	craneWg.Add(2)
+
+	go func() {
+		var err error
+		crane1, err = NewCrane(ship, connectedHub, nil)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not create crane1: %s", testID, err))
+		}
+		err = crane1.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not start crane1: %s", testID, err))
+		}
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane2, err = NewCrane(ship.Reverse(), nil, identity)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not create crane2: %s", testID, err))
+		}
+		err = crane2.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("crane test %s could not start crane2: %s", testID, err))
+		}
+		craneWg.Done()
+	}()
+
+	craneWg.Wait()
+	t.Logf("crane test %s setup complete", testID)
+
+	// Wait async for test to complete, print stack after timeout.
+	finished := make(chan struct{})
+	go func() {
+		select {
+		case <-finished:
+		case <-time.After(10 * time.Second):
+			t.Logf("crane test %s is taking too long, print stack:", testID)
+			_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+			os.Exit(1)
+		}
+	}()
+
+	t.Logf("crane1 controller: %+v", crane1.Controller)
+	t.Logf("crane2 controller: %+v", crane2.Controller)
+
+	// Create terminals and run test.
+	st := &StreamingTerminal{
+		test:     t,
+		id:       8,
+		crane:    crane2,
+		recv:     make(chan *terminal.Msg),
+		testData: []byte("The quick brown fox jumps over the lazy dog."),
+	}
+	crane2.terminals[st.ID()] = st
+
+	// Run streaming test.
+	var streamingWg sync.WaitGroup
+	streamingWg.Add(2)
+	count := 10000
+	go func() {
+		for i := 1; i <= count; i++ {
+			msg := terminal.NewMsg(st.testData)
+			msg.FlowID = st.id
+			err := crane1.Send(msg, 1*time.Second)
+			if err != nil {
+				msg.Finish()
+				crane1.Stop(err.Wrap("failed to submit terminal msg"))
+			}
+			// log.Tracef("spn/testing: + %d", i)
+		}
+		t.Logf("crane test %s done with sending", testID)
+		streamingWg.Done()
+	}()
+	go func() {
+		for i := 1; i <= count; i++ {
+			msg := <-st.recv
+			assert.Equal(t, st.testData, msg.Data.CompileData(), "data mismatched")
+			// log.Tracef("spn/testing: - %d", i)
+		}
+		t.Logf("crane test %s done with receiving", testID)
+		streamingWg.Done()
+	}()
+
+	// Wait for completion.
+	streamingWg.Wait()
+	close(finished)
+}
+
+var testIdentity *cabin.Identity
+
+func getTestIdentity(t *testing.T) (*cabin.Identity, *hub.Hub) {
+	t.Helper()
+
+	if testIdentity == nil {
+		var err error
+		testIdentity, err = cabin.CreateIdentity(module.Ctx, "test")
+		if err != nil {
+			t.Fatalf("failed to create identity: %s", err)
+		}
+	}
+
+	return testIdentity, testIdentity.Hub
+}

spn/docks/crane_verify.go

@@ -0,0 +1,85 @@
+package docks
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/varint"
+	"github.com/safing/portmaster/spn/cabin"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	hubVerificationPurpose = "hub identify verification"
+)
+
+// VerifyConnectedHub verifies the connected Hub.
+func (crane *Crane) VerifyConnectedHub(callerCtx context.Context) error {
+	if !crane.ship.IsMine() || crane.nextTerminalID != 0 || crane.Public() {
+		return errors.New("hub verification can only be executed in init phase by the client")
+	}
+
+	// Create verification request.
+	v, request, err := cabin.CreateVerificationRequest(hubVerificationPurpose, "", "")
+	if err != nil {
+		return fmt.Errorf("failed to create verification request: %w", err)
+	}
+
+	// Send it.
+	msg := container.New(
+		varint.Pack8(CraneMsgTypeVerify),
+		request,
+	)
+	msg.PrependLength()
+	err = crane.ship.Load(msg.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send verification request: %w", err)
+	}
+
+	// Wait for reply.
+	var reply *container.Container
+	select {
+	case reply = <-crane.unloading:
+	case <-time.After(2 * time.Minute):
+		// Use a big timeout here, as this might keep servers from joining the
+		// network at all, as every servers needs to verify every server, no
+		// matter how far away.
+		return terminal.ErrTimeout.With("waiting for verification reply")
+	case <-crane.ctx.Done():
+		return terminal.ErrShipSunk.With("waiting for verification reply")
+	case <-callerCtx.Done():
+		return terminal.ErrShipSunk.With("waiting for verification reply")
+	}
+
+	// Verify reply.
+	return v.Verify(reply.CompileData(), crane.ConnectedHub)
+}
+
+func (crane *Crane) handleCraneVerification(request *container.Container) *terminal.Error {
+	// Check if we have an identity.
+	if crane.identity == nil {
+		return terminal.ErrIncorrectUsage.With("cannot handle verification request without designated identity")
+	}
+
+	response, err := crane.identity.SignVerificationRequest(
+		request.CompileData(),
+		hubVerificationPurpose,
+		"", "",
+	)
+	if err != nil {
+		return terminal.ErrInternalError.With("failed to sign verification request: %w", err)
+	}
+	msg := container.New(response)
+
+	// Manually send reply.
+	msg.PrependLength()
+	err = crane.ship.Load(msg.CompileData())
+	if err != nil {
+		return terminal.ErrShipSunk.With("failed to send verification reply: %w", err)
+	}
+
+	return nil
+}

spn/docks/cranehooks.go

@@ -0,0 +1,46 @@
+package docks
+
+import (
+	"sync"
+
+	"github.com/safing/portbase/log"
+)
+
+var (
+	craneUpdateHook     func(crane *Crane)
+	craneUpdateHookLock sync.Mutex
+)
+
+// RegisterCraneUpdateHook allows the captain to hook into receiving updates for cranes.
+func RegisterCraneUpdateHook(fn func(crane *Crane)) {
+	craneUpdateHookLock.Lock()
+	defer craneUpdateHookLock.Unlock()
+
+	if craneUpdateHook == nil {
+		craneUpdateHook = fn
+	} else {
+		log.Error("spn/docks: crane update hook already registered")
+	}
+}
+
+// ResetCraneUpdateHook resets the hook for receiving updates for cranes.
+func ResetCraneUpdateHook() {
+	craneUpdateHookLock.Lock()
+	defer craneUpdateHookLock.Unlock()
+
+	craneUpdateHook = nil
+}
+
+// NotifyUpdate calls the registers crane update hook function.
+func (crane *Crane) NotifyUpdate() {
+	if crane == nil {
+		return
+	}
+
+	craneUpdateHookLock.Lock()
+	defer craneUpdateHookLock.Unlock()
+
+	if craneUpdateHook != nil {
+		craneUpdateHook(crane)
+	}
+}

spn/docks/hub_import.go

@@ -0,0 +1,189 @@
+package docks
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/spn/conf"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/ships"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+var hubImportLock sync.Mutex
+
+// ImportAndVerifyHubInfo imports the given hub message and verifies them.
+func ImportAndVerifyHubInfo(ctx context.Context, hubID string, announcementData, statusData []byte, mapName string, scope hub.Scope) (h *hub.Hub, forward bool, tErr *terminal.Error) {
+	var firstErr *terminal.Error
+
+	// Synchronize import, as we might easily learn of a new hub from different
+	// gossip channels simultaneously.
+	hubImportLock.Lock()
+	defer hubImportLock.Unlock()
+
+	// Check arguments.
+	if announcementData == nil && statusData == nil {
+		return nil, false, terminal.ErrInternalError.With("no announcement or status supplied")
+	}
+
+	// Import Announcement, if given.
+	var hubKnown, hubChanged bool
+	if announcementData != nil {
+		hubFromMsg, known, changed, err := hub.ApplyAnnouncement(nil, announcementData, mapName, scope, false)
+		if err != nil && firstErr == nil {
+			firstErr = terminal.ErrInternalError.With("failed to apply announcement: %w", err)
+		}
+		if known {
+			hubKnown = true
+		}
+		if changed {
+			hubChanged = true
+		}
+		if hubFromMsg != nil {
+			h = hubFromMsg
+		}
+	}
+
+	// Import Status, if given.
+	if statusData != nil {
+		hubFromMsg, known, changed, err := hub.ApplyStatus(h, statusData, mapName, scope, false)
+		if err != nil && firstErr == nil {
+			firstErr = terminal.ErrInternalError.With("failed to apply status: %w", err)
+		}
+		if known && announcementData == nil {
+			// If we parsed an announcement before, "known" will always be true here,
+			// as we supply hub.ApplyStatus with a hub.
+			hubKnown = true
+		}
+		if changed {
+			hubChanged = true
+		}
+		if hubFromMsg != nil {
+			h = hubFromMsg
+		}
+	}
+
+	// Only continue if we now have a Hub.
+	if h == nil {
+		if firstErr != nil {
+			return nil, false, firstErr
+		}
+		return nil, false, terminal.ErrInternalError.With("got not hub after data import")
+	}
+
+	// Abort if the given hub ID does not match.
+	// We may have just connected to the wrong IP address.
+	if hubID != "" && h.ID != hubID {
+		return nil, false, terminal.ErrInternalError.With("hub mismatch")
+	}
+
+	// Verify hub if:
+	// - There is no error up until here.
+	// - There has been any change.
+	// - The hub is not verified yet.
+	// - We're a public Hub.
+	// - We're not testing.
+	if firstErr == nil && hubChanged && !h.Verified() && conf.PublicHub() && !runningTests {
+		if !conf.HubHasIPv4() && !conf.HubHasIPv6() {
+			firstErr = terminal.ErrInternalError.With("no hub networks set")
+		}
+		if h.Info.IPv4 != nil && conf.HubHasIPv4() {
+			err := verifyHubIP(ctx, h, h.Info.IPv4)
+			if err != nil {
+				firstErr = terminal.ErrIntegrity.With("failed to verify IPv4 address %s of %s: %w", h.Info.IPv4, h, err)
+			}
+		}
+		if h.Info.IPv6 != nil && conf.HubHasIPv6() {
+			err := verifyHubIP(ctx, h, h.Info.IPv6)
+			if err != nil {
+				firstErr = terminal.ErrIntegrity.With("failed to verify IPv6 address %s of %s: %w", h.Info.IPv6, h, err)
+			}
+		}
+
+		if firstErr != nil {
+			func() {
+				h.Lock()
+				defer h.Unlock()
+				h.InvalidInfo = true
+			}()
+			log.Warningf("spn/docks: failed to verify IPs of %s: %s", h, firstErr)
+		} else {
+			func() {
+				h.Lock()
+				defer h.Unlock()
+				h.VerifiedIPs = true
+			}()
+			log.Infof("spn/docks: verified IPs of %s: IPv4=%s IPv6=%s", h, h.Info.IPv4, h.Info.IPv6)
+		}
+	}
+
+	// Dismiss initial imports with errors.
+	if !hubKnown && firstErr != nil {
+		return nil, false, firstErr
+	}
+
+	// Don't do anything if nothing changed.
+	if !hubChanged {
+		return h, false, firstErr
+	}
+
+	// We now have one of:
+	// - A unknown Hub without error.
+	// - A known Hub without error.
+	// - A known Hub with error, which we want to save and propagate.
+
+	// Save the Hub to the database.
+	err := h.Save()
+	if err != nil {
+		log.Errorf("spn/docks: failed to persist %s: %s", h, err)
+	}
+
+	// Save the raw messages to the database.
+	if announcementData != nil {
+		err = hub.SaveHubMsg(h.ID, h.Map, hub.MsgTypeAnnouncement, announcementData)
+		if err != nil {
+			log.Errorf("spn/docks: failed to save raw announcement msg of %s: %s", h, err)
+		}
+	}
+	if statusData != nil {
+		err = hub.SaveHubMsg(h.ID, h.Map, hub.MsgTypeStatus, statusData)
+		if err != nil {
+			log.Errorf("spn/docks: failed to save raw status msg of %s: %s", h, err)
+		}
+	}
+
+	return h, true, firstErr
+}
+
+func verifyHubIP(ctx context.Context, h *hub.Hub, ip net.IP) error {
+	// Create connection.
+	ship, err := ships.Launch(ctx, h, nil, ip)
+	if err != nil {
+		return fmt.Errorf("failed to launch ship to %s: %w", ip, err)
+	}
+
+	// Start crane for receiving reply.
+	crane, err := NewCrane(ship, h, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create crane: %w", err)
+	}
+	module.StartWorker("crane unloader", crane.unloader)
+	defer crane.Stop(nil)
+
+	// Verify Hub.
+	err = crane.VerifyConnectedHub(ctx)
+	if err != nil {
+		return err
+	}
+
+	// End connection.
+	tErr := crane.endInit()
+	if tErr != nil {
+		log.Debugf("spn/docks: failed to end verification connection to %s: %s", ip, tErr)
+	}
+
+	return nil
+}

spn/docks/measurements.go

@@ -0,0 +1,108 @@
+package docks
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/ships"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// Measurement Configuration.
+const (
+	CraneMeasurementTTLDefault    = 30 * time.Minute
+	CraneMeasurementTTLByCostBase = 1 * time.Minute
+	CraneMeasurementTTLByCostMin  = 30 * time.Minute
+	CraneMeasurementTTLByCostMax  = 3 * time.Hour
+
+	// With a base TTL of 1m, this leads to:
+	// 20c     -> 20m -> raised to 30m
+	// 50c     -> 50m
+	// 100c    -> 1h40m
+	// 1000c   -> 16h40m -> capped to 3h.
+)
+
+// MeasureHub measures the connection to this Hub and saves the results to the
+// Hub.
+func MeasureHub(ctx context.Context, h *hub.Hub, checkExpiryWith time.Duration) *terminal.Error {
+	// Check if we are measuring before building a connection.
+	if capacityTestRunning.IsSet() {
+		return terminal.ErrTryAgainLater.With("another capacity op is already running")
+	}
+
+	// Check if we have a connection to this Hub.
+	crane := GetAssignedCrane(h.ID)
+	if crane == nil {
+		// Connect to Hub.
+		var err error
+		crane, err = establishCraneForMeasuring(ctx, h)
+		if err != nil {
+			return terminal.ErrConnectionError.With("failed to connect to %s: %s", h, err)
+		}
+		// Stop crane if established just for measuring.
+		defer crane.Stop(nil)
+	}
+
+	// Run latency test.
+	_, expires := h.GetMeasurements().GetLatency()
+	if checkExpiryWith == 0 || time.Now().Add(-checkExpiryWith).After(expires) {
+		latOp, tErr := NewLatencyTestOp(crane.Controller)
+		if !tErr.IsOK() {
+			return tErr
+		}
+		select {
+		case tErr = <-latOp.Result():
+			if !tErr.IsOK() {
+				return tErr
+			}
+		case <-ctx.Done():
+			return terminal.ErrCanceled
+		case <-time.After(1 * time.Minute):
+			crane.Controller.StopOperation(latOp, terminal.ErrTimeout)
+			return terminal.ErrTimeout.With("waiting for latency test")
+		}
+	}
+
+	// Run capacity test.
+	_, expires = h.GetMeasurements().GetCapacity()
+	if checkExpiryWith == 0 || time.Now().Add(-checkExpiryWith).After(expires) {
+		capOp, tErr := NewCapacityTestOp(crane.Controller, nil)
+		if !tErr.IsOK() {
+			return tErr
+		}
+		select {
+		case tErr = <-capOp.Result():
+			if !tErr.IsOK() {
+				return tErr
+			}
+		case <-ctx.Done():
+			return terminal.ErrCanceled
+		case <-time.After(1 * time.Minute):
+			crane.Controller.StopOperation(capOp, terminal.ErrTimeout)
+			return terminal.ErrTimeout.With("waiting for capacity test")
+		}
+	}
+
+	return nil
+}
+
+func establishCraneForMeasuring(ctx context.Context, dst *hub.Hub) (*Crane, error) {
+	ship, err := ships.Launch(ctx, dst, nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to launch ship: %w", err)
+	}
+
+	crane, err := NewCrane(ship, dst, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create crane: %w", err)
+	}
+
+	err = crane.Start(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start crane: %w", err)
+	}
+
+	return crane, nil
+}

spn/docks/metrics.go

@@ -0,0 +1,404 @@
+package docks
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/metrics"
+)
+
+var (
+	newCranes              *metrics.Counter
+	newPublicCranes        *metrics.Counter
+	newAuthenticatedCranes *metrics.Counter
+
+	trafficBytesPublicCranes        *metrics.Counter
+	trafficBytesAuthenticatedCranes *metrics.Counter
+	trafficBytesPrivateCranes       *metrics.Counter
+
+	newExpandOp                  *metrics.Counter
+	expandOpDurationHistogram    *metrics.Histogram
+	expandOpRelayedDataHistogram *metrics.Histogram
+
+	metricsRegistered = abool.New()
+)
+
+func registerMetrics() (err error) {
+	// Only register metrics once.
+	if !metricsRegistered.SetToIf(false, true) {
+		return nil
+	}
+
+	// Total Crane Stats.
+
+	newCranes, err = metrics.NewCounter(
+		"spn/cranes/total",
+		nil,
+		&metrics.Options{
+			Name:       "SPN New Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	newPublicCranes, err = metrics.NewCounter(
+		"spn/cranes/public/total",
+		nil,
+		&metrics.Options{
+			Name:       "SPN New Public Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	newAuthenticatedCranes, err = metrics.NewCounter(
+		"spn/cranes/authenticated/total",
+		nil,
+		&metrics.Options{
+			Name:       "SPN New Authenticated Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	// Active Crane Stats.
+
+	_, err = metrics.NewGauge(
+		"spn/cranes/active",
+		map[string]string{
+			"status": "public",
+		},
+		getActivePublicCranes,
+		&metrics.Options{
+			Name:       "SPN Active Public Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/cranes/active",
+		map[string]string{
+			"status": "authenticated",
+		},
+		getActiveAuthenticatedCranes,
+		&metrics.Options{
+			Name:       "SPN Active Authenticated Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/cranes/active",
+		map[string]string{
+			"status": "private",
+		},
+		getActivePrivateCranes,
+		&metrics.Options{
+			Name:       "SPN Active Private Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/cranes/active",
+		map[string]string{
+			"status": "stopping",
+		},
+		getActiveStoppingCranes,
+		&metrics.Options{
+			Name:       "SPN Active Stopping Cranes",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	// Crane Traffic Stats.
+
+	trafficBytesPublicCranes, err = metrics.NewCounter(
+		"spn/cranes/bytes",
+		map[string]string{
+			"status": "public",
+		},
+		&metrics.Options{
+			Name:       "SPN Public Crane Traffic",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	trafficBytesAuthenticatedCranes, err = metrics.NewCounter(
+		"spn/cranes/bytes",
+		map[string]string{
+			"status": "authenticated",
+		},
+		&metrics.Options{
+			Name:       "SPN Authenticated Crane Traffic",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	trafficBytesPrivateCranes, err = metrics.NewCounter(
+		"spn/cranes/bytes",
+		map[string]string{
+			"status": "private",
+		},
+		&metrics.Options{
+			Name:       "SPN Private Crane Traffic",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	// Lane Stats.
+
+	_, err = metrics.NewGauge(
+		"spn/lanes/latency/avg/seconds",
+		nil,
+		getAvgLaneLatencyStat,
+		&metrics.Options{
+			Name:       "SPN Avg Lane Latency",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/lanes/latency/min/seconds",
+		nil,
+		getMinLaneLatencyStat,
+		&metrics.Options{
+			Name:       "SPN Min Lane Latency",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/lanes/capacity/avg/bytes",
+		nil,
+		getAvgLaneCapacityStat,
+		&metrics.Options{
+			Name:       "SPN Avg Lane Capacity",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/lanes/capacity/max/bytes",
+		nil,
+		getMaxLaneCapacityStat,
+		&metrics.Options{
+			Name:       "SPN Max Lane Capacity",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	// Expand Op Stats.
+
+	newExpandOp, err = metrics.NewCounter(
+		"spn/op/expand/total",
+		nil,
+		&metrics.Options{
+			Name:       "SPN Total Expand Operations",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_, err = metrics.NewGauge(
+		"spn/op/expand/active",
+		nil,
+		getActiveExpandOpsStat,
+		&metrics.Options{
+			Name:       "SPN Active Expand Operations",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	expandOpDurationHistogram, err = metrics.NewHistogram(
+		"spn/op/expand/histogram/duration/seconds",
+		nil,
+		&metrics.Options{
+			Name:       "SPN Expand Operation Duration Histogram",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	expandOpRelayedDataHistogram, err = metrics.NewHistogram(
+		"spn/op/expand/histogram/traffic/bytes",
+		nil,
+		&metrics.Options{
+			Name:       "SPN Expand Operation Relayed Data Histogram",
+			Permission: api.PermitUser,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+func getActiveExpandOpsStat() float64 {
+	return float64(atomic.LoadInt64(activeExpandOps))
+}
+
+var (
+	craneStats        *craneGauges
+	craneStatsExpires time.Time
+	craneStatsLock    sync.Mutex
+	craneStatsTTL     = 55 * time.Second
+)
+
+type craneGauges struct {
+	publicActive        float64
+	authenticatedActive float64
+	privateActive       float64
+	stoppingActive      float64
+
+	laneLatencyAvg  float64
+	laneLatencyMin  float64
+	laneCapacityAvg float64
+	laneCapacityMax float64
+}
+
+func getActivePublicCranes() float64        { return getCraneStats().publicActive }
+func getActiveAuthenticatedCranes() float64 { return getCraneStats().authenticatedActive }
+func getActivePrivateCranes() float64       { return getCraneStats().privateActive }
+func getActiveStoppingCranes() float64      { return getCraneStats().stoppingActive }
+func getAvgLaneLatencyStat() float64        { return getCraneStats().laneLatencyAvg }
+func getMinLaneLatencyStat() float64        { return getCraneStats().laneLatencyMin }
+func getAvgLaneCapacityStat() float64       { return getCraneStats().laneCapacityAvg }
+func getMaxLaneCapacityStat() float64       { return getCraneStats().laneCapacityMax }
+
+func getCraneStats() *craneGauges {
+	craneStatsLock.Lock()
+	defer craneStatsLock.Unlock()
+
+	// Return cache if still valid.
+	if time.Now().Before(craneStatsExpires) {
+		return craneStats
+	}
+
+	// Refresh.
+	craneStats = &craneGauges{}
+	var laneStatCnt float64
+	for _, crane := range getAllCranes() {
+		switch {
+		case crane.Stopped():
+			continue
+		case crane.IsStopping():
+			craneStats.stoppingActive++
+			continue
+		case crane.Public():
+			craneStats.publicActive++
+		case crane.Authenticated():
+			craneStats.authenticatedActive++
+			continue
+		default:
+			craneStats.privateActive++
+			continue
+		}
+
+		// Get lane stats.
+		if crane.ConnectedHub == nil {
+			continue
+		}
+		measurements := crane.ConnectedHub.GetMeasurements()
+		laneLatency, _ := measurements.GetLatency()
+		if laneLatency == 0 {
+			continue
+		}
+		laneCapacity, _ := measurements.GetCapacity()
+		if laneCapacity == 0 {
+			continue
+		}
+
+		// Only use data if both latency and capacity is available.
+		laneStatCnt++
+
+		// Convert to base unit: seconds.
+		latency := laneLatency.Seconds()
+		// Add to avg and set min if lower.
+		craneStats.laneLatencyAvg += latency
+		if craneStats.laneLatencyMin > latency || craneStats.laneLatencyMin == 0 {
+			craneStats.laneLatencyMin = latency
+		}
+
+		// Convert in base unit: bytes.
+		capacity := float64(laneCapacity) / 8
+		// Add to avg and set max if higher.
+		craneStats.laneCapacityAvg += capacity
+		if craneStats.laneCapacityMax < capacity {
+			craneStats.laneCapacityMax = capacity
+		}
+	}
+
+	// Create averages.
+	if laneStatCnt > 0 {
+		craneStats.laneLatencyAvg /= laneStatCnt
+		craneStats.laneCapacityAvg /= laneStatCnt
+	}
+
+	craneStatsExpires = time.Now().Add(craneStatsTTL)
+	return craneStats
+}
+
+func (crane *Crane) submitCraneTrafficStats(bytes int) {
+	switch {
+	case crane.Stopped():
+		return
+	case crane.Public():
+		trafficBytesPublicCranes.Add(bytes)
+	case crane.Authenticated():
+		trafficBytesAuthenticatedCranes.Add(bytes)
+	default:
+		trafficBytesPrivateCranes.Add(bytes)
+	}
+}

spn/docks/module.go

@@ -0,0 +1,117 @@
+package docks
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portbase/rng"
+	_ "github.com/safing/portmaster/spn/access"
+)
+
+var (
+	module *modules.Module
+
+	allCranes      = make(map[string]*Crane) // ID = Crane ID
+	assignedCranes = make(map[string]*Crane) // ID = connected Hub ID
+	cranesLock     sync.RWMutex
+
+	runningTests bool
+)
+
+func init() {
+	module = modules.Register("docks", nil, start, stopAllCranes, "terminal", "cabin", "access")
+}
+
+func start() error {
+	return registerMetrics()
+}
+
+func registerCrane(crane *Crane) error {
+	cranesLock.Lock()
+	defer cranesLock.Unlock()
+
+	// Generate new IDs until a unique one is found.
+	for i := 0; i < 100; i++ {
+		// Generate random ID.
+		randomID, err := rng.Bytes(3)
+		if err != nil {
+			return fmt.Errorf("failed to generate crane ID: %w", err)
+		}
+		newID := hex.EncodeToString(randomID)
+
+		// Check if ID already exists.
+		_, ok := allCranes[newID]
+		if !ok {
+			crane.ID = newID
+			allCranes[crane.ID] = crane
+			return nil
+		}
+	}
+
+	return errors.New("failed to find unique crane ID")
+}
+
+func unregisterCrane(crane *Crane) {
+	cranesLock.Lock()
+	defer cranesLock.Unlock()
+
+	delete(allCranes, crane.ID)
+	if crane.ConnectedHub != nil {
+		delete(assignedCranes, crane.ConnectedHub.ID)
+	}
+}
+
+func stopAllCranes() error {
+	for _, crane := range getAllCranes() {
+		crane.Stop(nil)
+	}
+	return nil
+}
+
+// AssignCrane assigns a crane to the given Hub ID.
+func AssignCrane(hubID string, crane *Crane) {
+	cranesLock.Lock()
+	defer cranesLock.Unlock()
+
+	assignedCranes[hubID] = crane
+}
+
+// GetAssignedCrane returns the assigned crane of the given Hub ID.
+func GetAssignedCrane(hubID string) *Crane {
+	cranesLock.RLock()
+	defer cranesLock.RUnlock()
+
+	crane, ok := assignedCranes[hubID]
+	if ok {
+		return crane
+	}
+	return nil
+}
+
+func getAllCranes() map[string]*Crane {
+	copiedCranes := make(map[string]*Crane, len(allCranes))
+
+	cranesLock.RLock()
+	defer cranesLock.RUnlock()
+
+	for id, crane := range allCranes {
+		copiedCranes[id] = crane
+	}
+	return copiedCranes
+}
+
+// GetAllAssignedCranes returns a copy of the map of all assigned cranes.
+func GetAllAssignedCranes() map[string]*Crane {
+	copiedCranes := make(map[string]*Crane, len(assignedCranes))
+
+	cranesLock.RLock()
+	defer cranesLock.RUnlock()
+
+	for destination, crane := range assignedCranes {
+		copiedCranes[destination] = crane
+	}
+	return copiedCranes
+}

spn/docks/module_test.go

@@ -0,0 +1,16 @@
+package docks
+
+import (
+	"testing"
+
+	"github.com/safing/portmaster/service/core/pmtesting"
+	"github.com/safing/portmaster/spn/access"
+	"github.com/safing/portmaster/spn/conf"
+)
+
+func TestMain(m *testing.M) {
+	runningTests = true
+	conf.EnablePublicHub(true) // Make hub config available.
+	access.EnableTestMode()    // Register test zone instead of real ones.
+	pmtesting.TestMain(m, module)
+}

spn/docks/op_capacity.go

@@ -0,0 +1,356 @@
+package docks
+
+import (
+	"bytes"
+	"context"
+	"sync/atomic"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	// CapacityTestOpType is the type ID of the capacity test operation.
+	CapacityTestOpType = "capacity"
+
+	defaultCapacityTestVolume = 50000000  // 50MB
+	maxCapacityTestVolume     = 100000000 // 100MB
+
+	defaultCapacityTestMaxTime = 5 * time.Second
+	maxCapacityTestMaxTime     = 15 * time.Second
+	capacityTestTimeout        = 30 * time.Second
+
+	capacityTestMsgSize     = 1000
+	capacityTestSendTimeout = 1000 * time.Millisecond
+)
+
+var (
+	capacityTestSendData           = make([]byte, capacityTestMsgSize)
+	capacityTestDataReceivedSignal = []byte("ACK")
+
+	capacityTestRunning = abool.New()
+)
+
+// CapacityTestOp is used for capacity test operations.
+type CapacityTestOp struct { //nolint:maligned
+	terminal.OperationBase
+
+	opts *CapacityTestOptions
+
+	started       bool
+	startTime     time.Time
+	senderStarted bool
+
+	recvQueue              chan *terminal.Msg
+	dataReceived           int
+	dataReceivedAckWasAckd bool
+
+	dataSent        *int64
+	dataSentWasAckd *abool.AtomicBool
+
+	testResult int
+	result     chan *terminal.Error
+}
+
+// CapacityTestOptions holds options for the capacity test.
+type CapacityTestOptions struct {
+	TestVolume int
+	MaxTime    time.Duration
+	testing    bool
+}
+
+// Type returns the type ID.
+func (op *CapacityTestOp) Type() string {
+	return CapacityTestOpType
+}
+
+func init() {
+	terminal.RegisterOpType(terminal.OperationFactory{
+		Type:     CapacityTestOpType,
+		Requires: terminal.IsCraneController,
+		Start:    startCapacityTestOp,
+	})
+}
+
+// NewCapacityTestOp runs a capacity test.
+func NewCapacityTestOp(t terminal.Terminal, opts *CapacityTestOptions) (*CapacityTestOp, *terminal.Error) {
+	// Check options.
+	if opts == nil {
+		opts = &CapacityTestOptions{
+			TestVolume: defaultCapacityTestVolume,
+			MaxTime:    defaultCapacityTestMaxTime,
+		}
+	}
+
+	// Check if another test is already running.
+	if !opts.testing && !capacityTestRunning.SetToIf(false, true) {
+		return nil, terminal.ErrTryAgainLater.With("another capacity op is already running")
+	}
+
+	// Create and init.
+	op := &CapacityTestOp{
+		opts:            opts,
+		recvQueue:       make(chan *terminal.Msg),
+		dataSent:        new(int64),
+		dataSentWasAckd: abool.New(),
+		result:          make(chan *terminal.Error, 1),
+	}
+
+	// Make capacity test request.
+	request, err := dsd.Dump(op.opts, dsd.CBOR)
+	if err != nil {
+		capacityTestRunning.UnSet()
+		return nil, terminal.ErrInternalError.With("failed to serialize capactity test options: %w", err)
+	}
+
+	// Send test request.
+	tErr := t.StartOperation(op, container.New(request), 1*time.Second)
+	if tErr != nil {
+		capacityTestRunning.UnSet()
+		return nil, tErr
+	}
+
+	// Start handler.
+	module.StartWorker("op capacity handler", op.handler)
+
+	return op, nil
+}
+
+func startCapacityTestOp(t terminal.Terminal, opID uint32, data *container.Container) (terminal.Operation, *terminal.Error) {
+	// Check if another test is already running.
+	if !capacityTestRunning.SetToIf(false, true) {
+		return nil, terminal.ErrTryAgainLater.With("another capacity op is already running")
+	}
+
+	// Parse options.
+	opts := &CapacityTestOptions{}
+	_, err := dsd.Load(data.CompileData(), opts)
+	if err != nil {
+		capacityTestRunning.UnSet()
+		return nil, terminal.ErrMalformedData.With("failed to parse options: %w", err)
+	}
+
+	// Check options.
+	if opts.TestVolume > maxCapacityTestVolume {
+		capacityTestRunning.UnSet()
+		return nil, terminal.ErrInvalidOptions.With("maximum volume exceeded")
+	}
+	if opts.MaxTime > maxCapacityTestMaxTime {
+		capacityTestRunning.UnSet()
+		return nil, terminal.ErrInvalidOptions.With("maximum maxtime exceeded")
+	}
+
+	// Create operation.
+	op := &CapacityTestOp{
+		opts:            opts,
+		recvQueue:       make(chan *terminal.Msg, 1000),
+		dataSent:        new(int64),
+		dataSentWasAckd: abool.New(),
+		result:          make(chan *terminal.Error, 1),
+	}
+	op.InitOperationBase(t, opID)
+
+	// Start handler and sender.
+	op.senderStarted = true
+	module.StartWorker("op capacity handler", op.handler)
+	module.StartWorker("op capacity sender", op.sender)
+
+	return op, nil
+}
+
+func (op *CapacityTestOp) handler(ctx context.Context) error {
+	defer capacityTestRunning.UnSet()
+
+	returnErr := terminal.ErrStopping
+	defer func() {
+		// Linters don't get that returnErr is used when directly used as defer.
+		op.Stop(op, returnErr)
+	}()
+
+	var maxTestTimeReached <-chan time.Time
+	opTimeout := time.After(capacityTestTimeout)
+
+	// Setup unit handling
+	var msg *terminal.Msg
+	defer msg.Finish()
+
+	// Handle receives.
+	for {
+		msg.Finish()
+
+		select {
+		case <-ctx.Done():
+			returnErr = terminal.ErrCanceled
+			return nil
+
+		case <-opTimeout:
+			returnErr = terminal.ErrTimeout
+			return nil
+
+		case <-maxTestTimeReached:
+			returnErr = op.reportMeasuredCapacity()
+			return nil
+
+		case msg = <-op.recvQueue:
+			// Record start time and start sender.
+			if !op.started {
+				op.started = true
+				op.startTime = time.Now()
+				maxTestTimeReached = time.After(op.opts.MaxTime)
+				if !op.senderStarted {
+					op.senderStarted = true
+					module.StartWorker("op capacity sender", op.sender)
+				}
+			}
+
+			// Add to received data counter.
+			op.dataReceived += msg.Data.Length()
+
+			// Check if we received the data received signal.
+			if msg.Data.Length() == len(capacityTestDataReceivedSignal) &&
+				bytes.Equal(msg.Data.CompileData(), capacityTestDataReceivedSignal) {
+				op.dataSentWasAckd.Set()
+			}
+
+			// Send the data received signal when we received the full test volume.
+			if op.dataReceived >= op.opts.TestVolume && !op.dataReceivedAckWasAckd {
+				tErr := op.Send(op.NewMsg(capacityTestDataReceivedSignal), capacityTestSendTimeout)
+				if tErr != nil {
+					returnErr = tErr.Wrap("failed to send data received signal")
+					return nil
+				}
+				atomic.AddInt64(op.dataSent, int64(len(capacityTestDataReceivedSignal)))
+				op.dataReceivedAckWasAckd = true
+
+				// Flush last message.
+				op.Flush(10 * time.Second)
+			}
+
+			// Check if we can complete the test.
+			if op.dataReceivedAckWasAckd &&
+				op.dataSentWasAckd.IsSet() {
+				returnErr = op.reportMeasuredCapacity()
+				return nil
+			}
+		}
+	}
+}
+
+func (op *CapacityTestOp) sender(ctx context.Context) error {
+	for {
+		// Send next chunk.
+		msg := op.NewMsg(capacityTestSendData)
+		msg.Unit.MakeHighPriority()
+		tErr := op.Send(msg, capacityTestSendTimeout)
+		if tErr != nil {
+			op.Stop(op, tErr.Wrap("failed to send capacity test data"))
+			return nil
+		}
+
+		// Add to sent data counter and stop sending if sending is complete.
+		if atomic.AddInt64(op.dataSent, int64(len(capacityTestSendData))) >= int64(op.opts.TestVolume) {
+			return nil
+		}
+
+		// Check if we have received an ack.
+		if op.dataSentWasAckd.IsSet() {
+			return nil
+		}
+
+		// Check if op has ended.
+		if op.Stopped() {
+			return nil
+		}
+	}
+}
+
+func (op *CapacityTestOp) reportMeasuredCapacity() *terminal.Error {
+	// Calculate lane capacity and set it.
+	timeNeeded := time.Since(op.startTime)
+	if timeNeeded <= 0 {
+		timeNeeded = 1
+	}
+	duplexBits := float64((int64(op.dataReceived) + atomic.LoadInt64(op.dataSent)) * 8)
+	duplexNSBitRate := duplexBits / float64(timeNeeded)
+	bitRate := (duplexNSBitRate / 2) * float64(time.Second)
+	op.testResult = int(bitRate)
+
+	// Save the result to the crane.
+	if controller, ok := op.Terminal().(*CraneControllerTerminal); ok {
+		if controller.Crane.ConnectedHub != nil {
+			controller.Crane.ConnectedHub.GetMeasurements().SetCapacity(op.testResult)
+			log.Infof(
+				"docks: measured capacity to %s: %.2f Mbit/s (%.2fMB down / %.2fMB up in %s)",
+				controller.Crane.ConnectedHub,
+				float64(op.testResult)/1000000,
+				float64(op.dataReceived)/1000000,
+				float64(atomic.LoadInt64(op.dataSent))/1000000,
+				timeNeeded,
+			)
+			return nil
+		} else if controller.Crane.IsMine() {
+			return terminal.ErrInternalError.With("capacity operation was run on %s without a connected hub set", controller.Crane)
+		}
+	} else if !runningTests {
+		return terminal.ErrInternalError.With("capacity operation was run on terminal that is not a crane controller, but %T", op.Terminal())
+	}
+
+	return nil
+}
+
+// Deliver delivers a message.
+func (op *CapacityTestOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	// Optimized delivery with 1s timeout.
+	select {
+	case op.recvQueue <- msg:
+	default:
+		select {
+		case op.recvQueue <- msg:
+		case <-time.After(1 * time.Second):
+			msg.Finish()
+			return terminal.ErrTimeout
+		}
+	}
+	return nil
+}
+
+// HandleStop gives the operation the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Stop() instead.
+func (op *CapacityTestOp) HandleStop(tErr *terminal.Error) (errorToSend *terminal.Error) {
+	// Return result to waiting routine.
+	select {
+	case op.result <- tErr:
+	default:
+	}
+
+	// Drain the recvQueue to finish the message units.
+drain:
+	for {
+		select {
+		case msg := <-op.recvQueue:
+			msg.Finish()
+		default:
+			select {
+			case msg := <-op.recvQueue:
+				msg.Finish()
+			case <-time.After(3 * time.Millisecond):
+				// Give some additional time buffer to drain the queue.
+				break drain
+			}
+		}
+	}
+
+	// Return error as is.
+	return tErr
+}
+
+// Result returns the result (end error) of the operation.
+func (op *CapacityTestOp) Result() <-chan *terminal.Error {
+	return op.result
+}

spn/docks/op_capacity_test.go

@@ -0,0 +1,85 @@
+package docks
+
+import (
+	"testing"
+	"time"
+
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+var (
+	testCapacityTestVolume  = 1_000_000
+	testCapacitytestMaxTime = 1 * time.Second
+)
+
+func TestCapacityOp(t *testing.T) { //nolint:paralleltest // Performance test.
+	// Defaults.
+	testCapacityOp(t, &CapacityTestOptions{
+		TestVolume: testCapacityTestVolume,
+		MaxTime:    testCapacitytestMaxTime,
+		testing:    true,
+	})
+
+	// Hit max time first.
+	testCapacityOp(t, &CapacityTestOptions{
+		TestVolume: testCapacityTestVolume,
+		MaxTime:    100 * time.Millisecond,
+		testing:    true,
+	})
+
+	// Hit volume first.
+	testCapacityOp(t, &CapacityTestOptions{
+		TestVolume: 100_000,
+		MaxTime:    testCapacitytestMaxTime,
+		testing:    true,
+	})
+}
+
+func testCapacityOp(t *testing.T, opts *CapacityTestOptions) {
+	t.Helper()
+
+	var (
+		capTestDelay            = 5 * time.Millisecond
+		capTestQueueSize uint32 = 10
+	)
+
+	// Create test terminal pair.
+	a, b, err := terminal.NewSimpleTestTerminalPair(
+		capTestDelay,
+		int(capTestQueueSize),
+		&terminal.TerminalOpts{
+			FlowControl:     terminal.FlowControlDFQ,
+			FlowControlSize: capTestQueueSize,
+		},
+	)
+	if err != nil {
+		t.Fatalf("failed to create test terminal pair: %s", err)
+	}
+
+	// Grant permission for op on remote terminal and start op.
+	b.GrantPermission(terminal.IsCraneController)
+	op, tErr := NewCapacityTestOp(a, opts)
+	if tErr != nil {
+		t.Fatalf("failed to start op: %s", err)
+	}
+
+	// Wait for result and check error.
+	tErr = <-op.Result()
+	if !tErr.IsOK() {
+		t.Fatalf("op failed: %s", tErr)
+	}
+	t.Logf("measured capacity: %d bit/s", op.testResult)
+
+	// Calculate expected bandwidth.
+	expectedBitsPerSecond := float64(capacityTestMsgSize*8*int64(capTestQueueSize)) / float64(capTestDelay) * float64(time.Second)
+	t.Logf("expected capacity: %f bit/s", expectedBitsPerSecond)
+
+	// Check if measured bandwidth is within parameters.
+	if float64(op.testResult) > expectedBitsPerSecond*1.6 {
+		t.Fatal("measured capacity too high")
+	}
+	// TODO: Check if we can raise this to at least 90%.
+	if float64(op.testResult) < expectedBitsPerSecond*0.2 {
+		t.Fatal("measured capacity too low")
+	}
+}

spn/docks/op_expand.go

@@ -0,0 +1,393 @@
+package docks
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portmaster/spn/conf"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// ExpandOpType is the type ID of the expand operation.
+const ExpandOpType string = "expand"
+
+var activeExpandOps = new(int64)
+
+// ExpandOp is used to expand to another Hub.
+type ExpandOp struct {
+	terminal.OperationBase
+	opts *terminal.TerminalOpts
+
+	// ctx is the context of the Terminal.
+	ctx context.Context
+	// cancelCtx cancels ctx.
+	cancelCtx context.CancelFunc
+
+	dataRelayed *uint64
+	ended       *abool.AtomicBool
+
+	relayTerminal *ExpansionRelayTerminal
+
+	// flowControl holds the flow control system.
+	flowControl terminal.FlowControl
+	// deliverProxy is populated with the configured deliver function
+	deliverProxy func(msg *terminal.Msg) *terminal.Error
+	// recvProxy is populated with the configured recv function
+	recvProxy func() <-chan *terminal.Msg
+	// sendProxy is populated with the configured send function
+	sendProxy func(msg *terminal.Msg, timeout time.Duration)
+}
+
+// ExpansionRelayTerminal is a relay used for expansion.
+type ExpansionRelayTerminal struct {
+	terminal.BareTerminal
+
+	op *ExpandOp
+
+	id    uint32
+	crane *Crane
+
+	abandoning *abool.AtomicBool
+
+	// flowControl holds the flow control system.
+	flowControl terminal.FlowControl
+	// deliverProxy is populated with the configured deliver function
+	deliverProxy func(msg *terminal.Msg) *terminal.Error
+	// recvProxy is populated with the configured recv function
+	recvProxy func() <-chan *terminal.Msg
+	// sendProxy is populated with the configured send function
+	sendProxy func(msg *terminal.Msg, timeout time.Duration)
+}
+
+// Type returns the type ID.
+func (op *ExpandOp) Type() string {
+	return ExpandOpType
+}
+
+// ID returns the operation ID.
+func (t *ExpansionRelayTerminal) ID() uint32 {
+	return t.id
+}
+
+// Ctx returns the operation context.
+func (op *ExpandOp) Ctx() context.Context {
+	return op.ctx
+}
+
+// Ctx returns the relay terminal context.
+func (t *ExpansionRelayTerminal) Ctx() context.Context {
+	return t.op.ctx
+}
+
+// Deliver delivers a message to the relay operation.
+func (op *ExpandOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	return op.deliverProxy(msg)
+}
+
+// Deliver delivers a message to the relay terminal.
+func (t *ExpansionRelayTerminal) Deliver(msg *terminal.Msg) *terminal.Error {
+	return t.deliverProxy(msg)
+}
+
+// Flush writes all data in the queues.
+func (op *ExpandOp) Flush(timeout time.Duration) {
+	if op.flowControl != nil {
+		op.flowControl.Flush(timeout)
+	}
+}
+
+// Flush writes all data in the queues.
+func (t *ExpansionRelayTerminal) Flush(timeout time.Duration) {
+	if t.flowControl != nil {
+		t.flowControl.Flush(timeout)
+	}
+}
+
+func init() {
+	terminal.RegisterOpType(terminal.OperationFactory{
+		Type:     ExpandOpType,
+		Requires: terminal.MayExpand,
+		Start:    expand,
+	})
+}
+
+func expand(t terminal.Terminal, opID uint32, data *container.Container) (terminal.Operation, *terminal.Error) {
+	// Submit metrics.
+	newExpandOp.Inc()
+
+	// Check if we are running a public hub.
+	if !conf.PublicHub() {
+		return nil, terminal.ErrPermissionDenied.With("expanding is only allowed on public hubs")
+	}
+
+	// Parse destination hub ID.
+	dstData, err := data.GetNextBlock()
+	if err != nil {
+		return nil, terminal.ErrMalformedData.With("failed to parse destination: %w", err)
+	}
+
+	// Parse terminal options.
+	opts, tErr := terminal.ParseTerminalOpts(data)
+	if tErr != nil {
+		return nil, tErr.Wrap("failed to parse terminal options")
+	}
+
+	// Get crane with destination.
+	relayCrane := GetAssignedCrane(string(dstData))
+	if relayCrane == nil {
+		return nil, terminal.ErrHubUnavailable.With("no crane assigned to %q", string(dstData))
+	}
+
+	// TODO: Expand outside of hot path.
+
+	// Create operation and terminal.
+	op := &ExpandOp{
+		opts:        opts,
+		dataRelayed: new(uint64),
+		ended:       abool.New(),
+		relayTerminal: &ExpansionRelayTerminal{
+			crane:      relayCrane,
+			id:         relayCrane.getNextTerminalID(),
+			abandoning: abool.New(),
+		},
+	}
+	op.InitOperationBase(t, opID)
+	op.ctx, op.cancelCtx = context.WithCancel(t.Ctx())
+	op.relayTerminal.op = op
+
+	// Create flow control.
+	switch opts.FlowControl {
+	case terminal.FlowControlDFQ:
+		// Operation
+		op.flowControl = terminal.NewDuplexFlowQueue(op.ctx, opts.FlowControlSize, op.submitBackwardUpstream)
+		op.deliverProxy = op.flowControl.Deliver
+		op.recvProxy = op.flowControl.Receive
+		op.sendProxy = op.submitBackwardFlowControl
+		// Relay Terminal
+		op.relayTerminal.flowControl = terminal.NewDuplexFlowQueue(op.ctx, opts.FlowControlSize, op.submitForwardUpstream)
+		op.relayTerminal.deliverProxy = op.relayTerminal.flowControl.Deliver
+		op.relayTerminal.recvProxy = op.relayTerminal.flowControl.Receive
+		op.relayTerminal.sendProxy = op.submitForwardFlowControl
+	case terminal.FlowControlNone:
+		// Operation
+		deliverToOp := make(chan *terminal.Msg, opts.FlowControlSize)
+		op.deliverProxy = terminal.MakeDirectDeliveryDeliverFunc(op.ctx, deliverToOp)
+		op.recvProxy = terminal.MakeDirectDeliveryRecvFunc(deliverToOp)
+		op.sendProxy = op.submitBackwardUpstream
+		// Relay Terminal
+		deliverToRelay := make(chan *terminal.Msg, opts.FlowControlSize)
+		op.relayTerminal.deliverProxy = terminal.MakeDirectDeliveryDeliverFunc(op.ctx, deliverToRelay)
+		op.relayTerminal.recvProxy = terminal.MakeDirectDeliveryRecvFunc(deliverToRelay)
+		op.relayTerminal.sendProxy = op.submitForwardUpstream
+	case terminal.FlowControlDefault:
+		fallthrough
+	default:
+		return nil, terminal.ErrInternalError.With("unknown flow control type %d", opts.FlowControl)
+	}
+
+	// Establish terminal on destination.
+	newInitData, tErr := opts.Pack()
+	if tErr != nil {
+		return nil, terminal.ErrInternalError.With("failed to re-pack options: %w", err)
+	}
+	tErr = op.relayTerminal.crane.EstablishNewTerminal(op.relayTerminal, newInitData)
+	if tErr != nil {
+		return nil, tErr
+	}
+
+	// Start workers.
+	module.StartWorker("expand op forward relay", op.forwardHandler)
+	module.StartWorker("expand op backward relay", op.backwardHandler)
+	if op.flowControl != nil {
+		op.flowControl.StartWorkers(module, "expand op")
+	}
+	if op.relayTerminal.flowControl != nil {
+		op.relayTerminal.flowControl.StartWorkers(module, "expand op terminal")
+	}
+
+	return op, nil
+}
+
+func (op *ExpandOp) submitForwardFlowControl(msg *terminal.Msg, timeout time.Duration) {
+	err := op.relayTerminal.flowControl.Send(msg, timeout)
+	if err != nil {
+		msg.Finish()
+		op.Stop(op, err.Wrap("failed to submit to forward flow control"))
+	}
+}
+
+func (op *ExpandOp) submitBackwardFlowControl(msg *terminal.Msg, timeout time.Duration) {
+	err := op.flowControl.Send(msg, timeout)
+	if err != nil {
+		msg.Finish()
+		op.Stop(op, err.Wrap("failed to submit to backward flow control"))
+	}
+}
+
+func (op *ExpandOp) submitForwardUpstream(msg *terminal.Msg, timeout time.Duration) {
+	msg.FlowID = op.relayTerminal.id
+	if msg.Unit.IsHighPriority() && op.opts.UsePriorityDataMsgs {
+		msg.Type = terminal.MsgTypePriorityData
+	} else {
+		msg.Type = terminal.MsgTypeData
+	}
+	err := op.relayTerminal.crane.Send(msg, timeout)
+	if err != nil {
+		msg.Finish()
+		op.Stop(op, err.Wrap("failed to submit to forward upstream"))
+	}
+}
+
+func (op *ExpandOp) submitBackwardUpstream(msg *terminal.Msg, timeout time.Duration) {
+	msg.FlowID = op.relayTerminal.id
+	if msg.Unit.IsHighPriority() && op.opts.UsePriorityDataMsgs {
+		msg.Type = terminal.MsgTypePriorityData
+	} else {
+		msg.Type = terminal.MsgTypeData
+		msg.Unit.RemovePriority()
+	}
+	// Note: op.Send() will transform high priority units to priority data msgs.
+	err := op.Send(msg, timeout)
+	if err != nil {
+		msg.Finish()
+		op.Stop(op, err.Wrap("failed to submit to backward upstream"))
+	}
+}
+
+func (op *ExpandOp) forwardHandler(_ context.Context) error {
+	// Metrics setup and submitting.
+	atomic.AddInt64(activeExpandOps, 1)
+	started := time.Now()
+	defer func() {
+		atomic.AddInt64(activeExpandOps, -1)
+		expandOpDurationHistogram.UpdateDuration(started)
+		expandOpRelayedDataHistogram.Update(float64(atomic.LoadUint64(op.dataRelayed)))
+	}()
+
+	for {
+		select {
+		case msg := <-op.recvProxy():
+			// Debugging:
+			// log.Debugf("spn/testing: forwarding at %s: %s", op.FmtID(), spew.Sdump(c.CompileData()))
+
+			// Wait for processing slot.
+			msg.Unit.WaitForSlot()
+
+			// Count relayed data for metrics.
+			atomic.AddUint64(op.dataRelayed, uint64(msg.Data.Length()))
+
+			// Receive data from the origin and forward it to the relay.
+			op.relayTerminal.sendProxy(msg, 1*time.Minute)
+
+		case <-op.ctx.Done():
+			return nil
+		}
+	}
+}
+
+func (op *ExpandOp) backwardHandler(_ context.Context) error {
+	for {
+		select {
+		case msg := <-op.relayTerminal.recvProxy():
+			// Debugging:
+			// log.Debugf("spn/testing: backwarding at %s: %s", op.FmtID(), spew.Sdump(c.CompileData()))
+
+			// Wait for processing slot.
+			msg.Unit.WaitForSlot()
+
+			// Count relayed data for metrics.
+			atomic.AddUint64(op.dataRelayed, uint64(msg.Data.Length()))
+
+			// Receive data from the relay and forward it to the origin.
+			op.sendProxy(msg, 1*time.Minute)
+
+		case <-op.ctx.Done():
+			return nil
+		}
+	}
+}
+
+// HandleStop gives the operation the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Stop() instead.
+func (op *ExpandOp) HandleStop(err *terminal.Error) (errorToSend *terminal.Error) {
+	// Flush all messages before stopping.
+	op.Flush(1 * time.Minute)
+	op.relayTerminal.Flush(1 * time.Minute)
+
+	// Stop connected workers.
+	op.cancelCtx()
+
+	// Abandon connected terminal.
+	op.relayTerminal.Abandon(nil)
+
+	// Add context to error.
+	if err.IsError() {
+		return err.Wrap("relay operation failed with")
+	}
+	return err
+}
+
+// Abandon shuts down the terminal unregistering it from upstream and calling HandleAbandon().
+func (t *ExpansionRelayTerminal) Abandon(err *terminal.Error) {
+	if t.abandoning.SetToIf(false, true) {
+		module.StartWorker("terminal abandon procedure", func(_ context.Context) error {
+			t.handleAbandonProcedure(err)
+			return nil
+		})
+	}
+}
+
+// HandleAbandon gives the terminal the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Abandon() instead.
+func (t *ExpansionRelayTerminal) HandleAbandon(err *terminal.Error) (errorToSend *terminal.Error) {
+	// Stop the connected relay operation.
+	t.op.Stop(t.op, err)
+
+	// Add context to error.
+	if err.IsError() {
+		return err.Wrap("relay terminal failed with")
+	}
+	return err
+}
+
+// HandleDestruction gives the terminal the ability to clean up.
+// The terminal has already fully shut down at this point.
+// Should never be called directly. Call Abandon() instead.
+func (t *ExpansionRelayTerminal) HandleDestruction(err *terminal.Error) {}
+
+func (t *ExpansionRelayTerminal) handleAbandonProcedure(err *terminal.Error) {
+	// Call operation stop handle function for proper shutdown cleaning up.
+	err = t.HandleAbandon(err)
+
+	// Flush all messages before stopping.
+	t.Flush(1 * time.Minute)
+
+	// Send error to the connected Operation, if the error is internal.
+	if !err.IsExternal() {
+		if err == nil {
+			err = terminal.ErrStopping
+		}
+
+		msg := terminal.NewMsg(err.Pack())
+		msg.FlowID = t.ID()
+		msg.Type = terminal.MsgTypeStop
+		t.op.submitForwardUpstream(msg, 1*time.Second)
+	}
+}
+
+// FmtID returns the expansion ID hierarchy.
+func (op *ExpandOp) FmtID() string {
+	return fmt.Sprintf("%s>%d <r> %s#%d", op.Terminal().FmtID(), op.ID(), op.relayTerminal.crane.ID, op.relayTerminal.id)
+}
+
+// FmtID returns the expansion ID hierarchy.
+func (t *ExpansionRelayTerminal) FmtID() string {
+	return fmt.Sprintf("%s#%d", t.crane.ID, t.id)
+}

spn/docks/op_latency.go

@@ -0,0 +1,298 @@
+package docks
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/varint"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/rng"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	// LatencyTestOpType is the type ID of the latency test operation.
+	LatencyTestOpType = "latency"
+
+	latencyPingRequest  = 1
+	latencyPingResponse = 2
+
+	latencyTestNonceSize = 16
+	latencyTestRuns      = 10
+)
+
+var (
+	latencyTestPauseDuration = 1 * time.Second
+	latencyTestOpTimeout     = latencyTestRuns * latencyTestPauseDuration * 3
+)
+
+// LatencyTestOp is used to measure latency.
+type LatencyTestOp struct {
+	terminal.OperationBase
+}
+
+// LatencyTestClientOp is the client version of LatencyTestOp.
+type LatencyTestClientOp struct {
+	LatencyTestOp
+
+	lastPingSentAt    time.Time
+	lastPingNonce     []byte
+	measuredLatencies []time.Duration
+	responses         chan *terminal.Msg
+	testResult        time.Duration
+
+	result chan *terminal.Error
+}
+
+// Type returns the type ID.
+func (op *LatencyTestOp) Type() string {
+	return LatencyTestOpType
+}
+
+func init() {
+	terminal.RegisterOpType(terminal.OperationFactory{
+		Type:     LatencyTestOpType,
+		Requires: terminal.IsCraneController,
+		Start:    startLatencyTestOp,
+	})
+}
+
+// NewLatencyTestOp runs a latency test.
+func NewLatencyTestOp(t terminal.Terminal) (*LatencyTestClientOp, *terminal.Error) {
+	// Create and init.
+	op := &LatencyTestClientOp{
+		responses:         make(chan *terminal.Msg),
+		measuredLatencies: make([]time.Duration, 0, latencyTestRuns),
+		result:            make(chan *terminal.Error, 1),
+	}
+
+	// Make ping request.
+	pingRequest, err := op.createPingRequest()
+	if err != nil {
+		return nil, terminal.ErrInternalError.With("%w", err)
+	}
+
+	// Send ping.
+	tErr := t.StartOperation(op, pingRequest, 1*time.Second)
+	if tErr != nil {
+		return nil, tErr
+	}
+
+	// Start handler.
+	module.StartWorker("op latency handler", op.handler)
+
+	return op, nil
+}
+
+func (op *LatencyTestClientOp) handler(ctx context.Context) error {
+	returnErr := terminal.ErrStopping
+	defer func() {
+		// Linters don't get that returnErr is used when directly used as defer.
+		op.Stop(op, returnErr)
+	}()
+
+	var nextTest <-chan time.Time
+	opTimeout := time.After(latencyTestOpTimeout)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+
+		case <-opTimeout:
+			return nil
+
+		case <-nextTest:
+			// Create ping request msg.
+			pingRequest, err := op.createPingRequest()
+			if err != nil {
+				returnErr = terminal.ErrInternalError.With("%w", err)
+				return nil
+			}
+			msg := op.NewEmptyMsg()
+			msg.Unit.MakeHighPriority()
+			msg.Data = pingRequest
+
+			// Send it.
+			tErr := op.Send(msg, latencyTestOpTimeout)
+			if tErr != nil {
+				returnErr = tErr.Wrap("failed to send ping request")
+				return nil
+			}
+			op.Flush(1 * time.Second)
+
+			nextTest = nil
+
+		case msg := <-op.responses:
+			// Check if the op ended.
+			if msg == nil {
+				return nil
+			}
+
+			// Handle response
+			tErr := op.handleResponse(msg)
+			if tErr != nil {
+				returnErr = tErr
+				return nil //nolint:nilerr
+			}
+
+			// Check if we have enough latency tests.
+			if len(op.measuredLatencies) >= latencyTestRuns {
+				returnErr = op.reportMeasuredLatencies()
+				return nil
+			}
+
+			// Schedule next latency test, if not yet scheduled.
+			if nextTest == nil {
+				nextTest = time.After(latencyTestPauseDuration)
+			}
+		}
+	}
+}
+
+func (op *LatencyTestClientOp) createPingRequest() (*container.Container, error) {
+	// Generate nonce.
+	nonce, err := rng.Bytes(latencyTestNonceSize)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ping nonce")
+	}
+
+	// Set client request state.
+	op.lastPingSentAt = time.Now()
+	op.lastPingNonce = nonce
+
+	return container.New(
+		varint.Pack8(latencyPingRequest),
+		nonce,
+	), nil
+}
+
+func (op *LatencyTestClientOp) handleResponse(msg *terminal.Msg) *terminal.Error {
+	defer msg.Finish()
+
+	rType, err := msg.Data.GetNextN8()
+	if err != nil {
+		return terminal.ErrMalformedData.With("failed to get response type: %w", err)
+	}
+
+	switch rType {
+	case latencyPingResponse:
+		// Check if the ping nonce matches.
+		if !bytes.Equal(op.lastPingNonce, msg.Data.CompileData()) {
+			return terminal.ErrIntegrity.With("ping nonce mismatch")
+		}
+		op.lastPingNonce = nil
+		// Save latency.
+		op.measuredLatencies = append(op.measuredLatencies, time.Since(op.lastPingSentAt))
+
+		return nil
+	default:
+		return terminal.ErrIncorrectUsage.With("unknown response type")
+	}
+}
+
+func (op *LatencyTestClientOp) reportMeasuredLatencies() *terminal.Error {
+	// Find lowest value.
+	lowestLatency := time.Hour
+	for _, latency := range op.measuredLatencies {
+		if latency < lowestLatency {
+			lowestLatency = latency
+		}
+	}
+	op.testResult = lowestLatency
+
+	// Save the result to the crane.
+	if controller, ok := op.Terminal().(*CraneControllerTerminal); ok {
+		if controller.Crane.ConnectedHub != nil {
+			controller.Crane.ConnectedHub.GetMeasurements().SetLatency(op.testResult)
+			log.Infof("spn/docks: measured latency to %s: %s", controller.Crane.ConnectedHub, op.testResult)
+			return nil
+		} else if controller.Crane.IsMine() {
+			return terminal.ErrInternalError.With("latency operation was run on %s without a connected hub set", controller.Crane)
+		}
+	} else if !runningTests {
+		return terminal.ErrInternalError.With("latency operation was run on terminal that is not a crane controller, but %T", op.Terminal())
+	}
+	return nil
+}
+
+// Deliver delivers a message to the operation.
+func (op *LatencyTestClientOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	// Optimized delivery with 1s timeout.
+	select {
+	case op.responses <- msg:
+	default:
+		select {
+		case op.responses <- msg:
+		case <-time.After(1 * time.Second):
+			return terminal.ErrTimeout
+		}
+	}
+	return nil
+}
+
+// HandleStop gives the operation the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Stop() instead.
+func (op *LatencyTestClientOp) HandleStop(tErr *terminal.Error) (errorToSend *terminal.Error) {
+	close(op.responses)
+	select {
+	case op.result <- tErr:
+	default:
+	}
+	return tErr
+}
+
+// Result returns the result (end error) of the operation.
+func (op *LatencyTestClientOp) Result() <-chan *terminal.Error {
+	return op.result
+}
+
+func startLatencyTestOp(t terminal.Terminal, opID uint32, data *container.Container) (terminal.Operation, *terminal.Error) {
+	// Create operation.
+	op := &LatencyTestOp{}
+	op.InitOperationBase(t, opID)
+
+	// Handle first request.
+	msg := op.NewEmptyMsg()
+	msg.Data = data
+	tErr := op.Deliver(msg)
+	if tErr != nil {
+		return nil, tErr
+	}
+
+	return op, nil
+}
+
+// Deliver delivers a message to the operation.
+func (op *LatencyTestOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	// Get request type.
+	rType, err := msg.Data.GetNextN8()
+	if err != nil {
+		return terminal.ErrMalformedData.With("failed to get response type: %w", err)
+	}
+
+	switch rType {
+	case latencyPingRequest:
+		// Keep the nonce and just replace the msg type.
+		msg.Data.PrependNumber(latencyPingResponse)
+		msg.Type = terminal.MsgTypeData
+		msg.Unit.ReUse()
+		msg.Unit.MakeHighPriority()
+
+		// Send response.
+		tErr := op.Send(msg, latencyTestOpTimeout)
+		if tErr != nil {
+			return tErr.Wrap("failed to send ping response")
+		}
+		op.Flush(1 * time.Second)
+
+		return nil
+
+	default:
+		return terminal.ErrIncorrectUsage.With("unknown request type")
+	}
+}

spn/docks/op_latency_test.go

@@ -0,0 +1,59 @@
+package docks
+
+import (
+	"testing"
+	"time"
+
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+func TestLatencyOp(t *testing.T) {
+	t.Parallel()
+
+	var (
+		latTestDelay            = 10 * time.Millisecond
+		latTestQueueSize uint32 = 10
+	)
+
+	// Reduce waiting time.
+	latencyTestPauseDuration = 100 * time.Millisecond
+
+	// Create test terminal pair.
+	a, b, err := terminal.NewSimpleTestTerminalPair(
+		latTestDelay,
+		int(latTestQueueSize),
+		&terminal.TerminalOpts{
+			FlowControl:     terminal.FlowControlNone,
+			FlowControlSize: latTestQueueSize,
+		},
+	)
+	if err != nil {
+		t.Fatalf("failed to create test terminal pair: %s", err)
+	}
+
+	// Grant permission for op on remote terminal and start op.
+	b.GrantPermission(terminal.IsCraneController)
+	op, tErr := NewLatencyTestOp(a)
+	if tErr != nil {
+		t.Fatalf("failed to start op: %s", err)
+	}
+
+	// Wait for result and check error.
+	tErr = <-op.Result()
+	if tErr.IsError() {
+		t.Fatalf("op failed: %s", tErr)
+	}
+	t.Logf("measured latency: %f ms", float64(op.testResult)/float64(time.Millisecond))
+
+	// Calculate expected latency.
+	expectedLatency := float64(latTestDelay * 2)
+	t.Logf("expected latency: %f ms", expectedLatency/float64(time.Millisecond))
+
+	// Check if measured latency is within parameters.
+	if float64(op.testResult) > expectedLatency*1.2 {
+		t.Fatal("measured latency too high")
+	}
+	if float64(op.testResult) < expectedLatency*0.9 {
+		t.Fatal("measured latency too low")
+	}
+}

spn/docks/op_sync_state.go

@@ -0,0 +1,150 @@
+package docks
+
+import (
+	"context"
+	"time"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portmaster/spn/conf"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// SyncStateOpType is the type ID of the sync state operation.
+const SyncStateOpType = "sync/state"
+
+// SyncStateOp is used to sync the crane state.
+type SyncStateOp struct {
+	terminal.OneOffOperationBase
+}
+
+// SyncStateMessage holds the sync data.
+type SyncStateMessage struct {
+	Stopping        bool
+	RequestStopping bool
+}
+
+// Type returns the type ID.
+func (op *SyncStateOp) Type() string {
+	return SyncStateOpType
+}
+
+func init() {
+	terminal.RegisterOpType(terminal.OperationFactory{
+		Type:     SyncStateOpType,
+		Requires: terminal.IsCraneController,
+		Start:    runSyncStateOp,
+	})
+}
+
+// startSyncStateOp starts a worker that runs the sync state operation.
+func (crane *Crane) startSyncStateOp() {
+	module.StartWorker("sync crane state", func(ctx context.Context) error {
+		tErr := crane.Controller.SyncState(ctx)
+		if tErr != nil {
+			return tErr
+		}
+
+		return nil
+	})
+}
+
+// SyncState runs a sync state operation.
+func (controller *CraneControllerTerminal) SyncState(ctx context.Context) *terminal.Error {
+	// Check if we are a public Hub, whether we own the crane and whether the lane is public too.
+	if !conf.PublicHub() || !controller.Crane.Public() {
+		return nil
+	}
+
+	// Create and init.
+	op := &SyncStateOp{}
+	op.Init()
+
+	// Get optimization states.
+	requestStopping := false
+	func() {
+		controller.Crane.NetState.lock.Lock()
+		defer controller.Crane.NetState.lock.Unlock()
+
+		requestStopping = controller.Crane.NetState.stoppingRequested
+	}()
+
+	// Create sync message.
+	msg := &SyncStateMessage{
+		Stopping:        controller.Crane.stopping.IsSet(),
+		RequestStopping: requestStopping,
+	}
+	data, err := dsd.Dump(msg, dsd.CBOR)
+	if err != nil {
+		return terminal.ErrInternalError.With("%w", err)
+	}
+
+	// Send message.
+	tErr := controller.StartOperation(op, container.New(data), 30*time.Second)
+	if tErr != nil {
+		return tErr
+	}
+
+	// Wait for reply
+	select {
+	case tErr = <-op.Result:
+		if tErr.IsError() {
+			return tErr
+		}
+		return nil
+	case <-ctx.Done():
+		return nil
+	case <-time.After(1 * time.Minute):
+		return terminal.ErrTimeout.With("timed out while waiting for sync crane result")
+	}
+}
+
+func runSyncStateOp(t terminal.Terminal, opID uint32, data *container.Container) (terminal.Operation, *terminal.Error) {
+	// Check if we are a on a crane controller.
+	var ok bool
+	var controller *CraneControllerTerminal
+	if controller, ok = t.(*CraneControllerTerminal); !ok {
+		return nil, terminal.ErrIncorrectUsage.With("can only be used with a crane controller")
+	}
+
+	// Check if we are a public Hub and whether the lane is public too.
+	if !conf.PublicHub() || !controller.Crane.Public() {
+		return nil, terminal.ErrPermissionDenied.With("only public lanes can sync crane status")
+	}
+
+	// Load message.
+	syncState := &SyncStateMessage{}
+	_, err := dsd.Load(data.CompileData(), syncState)
+	if err != nil {
+		return nil, terminal.ErrMalformedData.With("failed to load sync state message: %w", err)
+	}
+
+	// Apply optimization state.
+	controller.Crane.NetState.lock.Lock()
+	defer controller.Crane.NetState.lock.Unlock()
+	controller.Crane.NetState.stoppingRequestedByPeer = syncState.RequestStopping
+
+	// Apply crane state only when we don't own the crane.
+	if !controller.Crane.IsMine() {
+		// Apply sync state.
+		var changed bool
+		if syncState.Stopping {
+			if controller.Crane.stopping.SetToIf(false, true) {
+				controller.Crane.NetState.markedStoppingAt = time.Now()
+				changed = true
+			}
+		} else {
+			if controller.Crane.stopping.SetToIf(true, false) {
+				controller.Crane.NetState.markedStoppingAt = time.Time{}
+				changed = true
+			}
+		}
+
+		// Notify of change.
+		if changed {
+			controller.Crane.NotifyUpdate()
+		}
+	}
+
+	return nil, nil
+}

spn/docks/op_whoami.go

@@ -0,0 +1,135 @@
+package docks
+
+import (
+	"time"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const (
+	// WhoAmIType is the type ID of the latency test operation.
+	WhoAmIType = "whoami"
+
+	whoAmITimeout = 3 * time.Second
+)
+
+// WhoAmIOp is used to request some metadata about the other side.
+type WhoAmIOp struct {
+	terminal.OneOffOperationBase
+
+	response *WhoAmIResponse
+}
+
+// WhoAmIResponse is a whoami response.
+type WhoAmIResponse struct {
+	// Timestamp in nanoseconds
+	Timestamp int64 `cbor:"t,omitempty" json:"t,omitempty"`
+
+	// Addr is the remote address as reported by the crane terminal (IP and port).
+	Addr string `cbor:"a,omitempty" json:"a,omitempty"`
+}
+
+// Type returns the type ID.
+func (op *WhoAmIOp) Type() string {
+	return WhoAmIType
+}
+
+func init() {
+	terminal.RegisterOpType(terminal.OperationFactory{
+		Type:  WhoAmIType,
+		Start: startWhoAmI,
+	})
+}
+
+// WhoAmI executes a whoami operation and returns the response.
+func WhoAmI(t terminal.Terminal) (*WhoAmIResponse, *terminal.Error) {
+	whoami, err := NewWhoAmIOp(t)
+	if err.IsError() {
+		return nil, err
+	}
+
+	// Wait for response.
+	select {
+	case tErr := <-whoami.Result:
+		if tErr.IsError() {
+			return nil, tErr
+		}
+		return whoami.response, nil
+	case <-time.After(whoAmITimeout * 2):
+		return nil, terminal.ErrTimeout
+	}
+}
+
+// NewWhoAmIOp starts a new whoami operation.
+func NewWhoAmIOp(t terminal.Terminal) (*WhoAmIOp, *terminal.Error) {
+	// Create operation and init.
+	op := &WhoAmIOp{}
+	op.OneOffOperationBase.Init()
+
+	// Send ping.
+	tErr := t.StartOperation(op, nil, whoAmITimeout)
+	if tErr != nil {
+		return nil, tErr
+	}
+
+	return op, nil
+}
+
+// Deliver delivers a message to the operation.
+func (op *WhoAmIOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	defer msg.Finish()
+
+	// Parse response.
+	response := &WhoAmIResponse{}
+	_, err := dsd.Load(msg.Data.CompileData(), response)
+	if err != nil {
+		return terminal.ErrMalformedData.With("failed to parse ping response: %w", err)
+	}
+
+	op.response = response
+	return terminal.ErrExplicitAck
+}
+
+func startWhoAmI(t terminal.Terminal, opID uint32, data *container.Container) (terminal.Operation, *terminal.Error) {
+	// Get crane terminal, if available.
+	ct, _ := t.(*CraneTerminal)
+
+	// Create response.
+	r := &WhoAmIResponse{
+		Timestamp: time.Now().UnixNano(),
+	}
+	if ct != nil {
+		r.Addr = ct.RemoteAddr().String()
+	}
+	response, err := dsd.Dump(r, dsd.CBOR)
+	if err != nil {
+		return nil, terminal.ErrInternalError.With("failed to create whoami response: %w", err)
+	}
+
+	// Send response.
+	msg := terminal.NewMsg(response)
+	msg.FlowID = opID
+	msg.Unit.MakeHighPriority()
+	if terminal.UsePriorityDataMsgs {
+		msg.Type = terminal.MsgTypePriorityData
+	}
+	tErr := t.Send(msg, whoAmITimeout)
+	if tErr != nil {
+		// Finish message unit on failure.
+		msg.Finish()
+		return nil, tErr.With("failed to send ping response")
+	}
+
+	// Operation is just one response and finished successfully.
+	return nil, nil
+}
+
+// HandleStop gives the operation the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Stop() instead.
+func (op *WhoAmIOp) HandleStop(err *terminal.Error) (errorToSend *terminal.Error) {
+	// Continue with usual handling of inherited base.
+	return op.OneOffOperationBase.HandleStop(err)
+}

spn/docks/op_whoami_test.go

@@ -0,0 +1,24 @@
+package docks
+
+import (
+	"testing"
+
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+func TestWhoAmIOp(t *testing.T) {
+	t.Parallel()
+
+	// Create test terminal pair.
+	a, _, err := terminal.NewSimpleTestTerminalPair(0, 0, nil)
+	if err != nil {
+		t.Fatalf("failed to create test terminal pair: %s", err)
+	}
+
+	// Run op.
+	resp, tErr := WhoAmI(a)
+	if tErr.IsError() {
+		t.Fatal(tErr)
+	}
+	t.Logf("whoami: %+v", resp)
+}

spn/docks/terminal_expansion.go

@@ -0,0 +1,150 @@
+package docks
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/container"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+// ExpansionTerminal is used for expanding to another Hub.
+type ExpansionTerminal struct {
+	*terminal.TerminalBase
+
+	relayOp *ExpansionTerminalRelayOp
+
+	changeNotifyFuncReady *abool.AtomicBool
+	changeNotifyFunc      func()
+
+	reachableChecked time.Time
+	reachableLock    sync.Mutex
+}
+
+// ExpansionTerminalRelayOp is the operation that connects to the relay.
+type ExpansionTerminalRelayOp struct {
+	terminal.OperationBase
+
+	expansionTerminal *ExpansionTerminal
+}
+
+// Type returns the type ID.
+func (op *ExpansionTerminalRelayOp) Type() string {
+	return ExpandOpType
+}
+
+// ExpandTo initiates an expansion.
+func ExpandTo(from terminal.Terminal, routeTo string, encryptFor *hub.Hub) (*ExpansionTerminal, *terminal.Error) {
+	// First, create the local endpoint terminal to generate the init data.
+
+	// Create options and bare expansion terminal.
+	opts := terminal.DefaultExpansionTerminalOpts()
+	opts.Encrypt = encryptFor != nil
+	expansion := &ExpansionTerminal{
+		changeNotifyFuncReady: abool.New(),
+	}
+	expansion.relayOp = &ExpansionTerminalRelayOp{
+		expansionTerminal: expansion,
+	}
+
+	// Create base terminal for expansion.
+	base, initData, tErr := terminal.NewLocalBaseTerminal(
+		module.Ctx,
+		0, // Ignore; The ID of the operation is used for communication.
+		from.FmtID(),
+		encryptFor,
+		opts,
+		expansion.relayOp,
+	)
+	if tErr != nil {
+		return nil, tErr.Wrap("failed to create expansion terminal base")
+	}
+	expansion.TerminalBase = base
+	base.SetTerminalExtension(expansion)
+	base.SetTimeout(defaultTerminalIdleTimeout)
+
+	// Second, start the actual relay operation.
+
+	// Create setup message for relay operation.
+	opInitData := container.New()
+	opInitData.AppendAsBlock([]byte(routeTo))
+	opInitData.AppendContainer(initData)
+
+	// Start relay operation on connected Hub.
+	tErr = from.StartOperation(expansion.relayOp, opInitData, 5*time.Second)
+	if tErr != nil {
+		return nil, tErr.Wrap("failed to start expansion operation")
+	}
+
+	// Start Workers.
+	base.StartWorkers(module, "expansion terminal")
+
+	return expansion, nil
+}
+
+// SetChangeNotifyFunc sets a callback function that is called when the terminal state changes.
+func (t *ExpansionTerminal) SetChangeNotifyFunc(f func()) {
+	if t.changeNotifyFuncReady.IsSet() {
+		return
+	}
+	t.changeNotifyFunc = f
+	t.changeNotifyFuncReady.Set()
+}
+
+// NeedsReachableCheck returns whether the terminal should be checked if it is
+// reachable via the existing network internal relayed connection.
+func (t *ExpansionTerminal) NeedsReachableCheck(maxCheckAge time.Duration) bool {
+	t.reachableLock.Lock()
+	defer t.reachableLock.Unlock()
+
+	return time.Since(t.reachableChecked) > maxCheckAge
+}
+
+// MarkReachable marks the terminal as reachable via the existing network
+// internal relayed connection.
+func (t *ExpansionTerminal) MarkReachable() {
+	t.reachableLock.Lock()
+	defer t.reachableLock.Unlock()
+
+	t.reachableChecked = time.Now()
+}
+
+// HandleDestruction gives the terminal the ability to clean up.
+// The terminal has already fully shut down at this point.
+// Should never be called directly. Call Abandon() instead.
+func (t *ExpansionTerminal) HandleDestruction(err *terminal.Error) {
+	// Trigger update of connected Pin.
+	if t.changeNotifyFuncReady.IsSet() {
+		t.changeNotifyFunc()
+	}
+
+	// Stop the relay operation.
+	// The error message is arlready sent by the terminal.
+	t.relayOp.Stop(t.relayOp, nil)
+}
+
+// CustomIDFormat formats the terminal ID.
+func (t *ExpansionTerminal) CustomIDFormat() string {
+	return fmt.Sprintf("%s~%d", t.relayOp.Terminal().FmtID(), t.relayOp.ID())
+}
+
+// Deliver delivers a message to the operation.
+func (op *ExpansionTerminalRelayOp) Deliver(msg *terminal.Msg) *terminal.Error {
+	// Proxy directly to expansion terminal.
+	return op.expansionTerminal.Deliver(msg)
+}
+
+// HandleStop gives the operation the ability to cleanly shut down.
+// The returned error is the error to send to the other side.
+// Should never be called directly. Call Stop() instead.
+func (op *ExpansionTerminalRelayOp) HandleStop(err *terminal.Error) (errorToSend *terminal.Error) {
+	// Stop the expansion terminal.
+	// The error message will be sent by the operation.
+	op.expansionTerminal.Abandon(nil)
+
+	return err
+}

spn/docks/terminal_expansion_test.go

@@ -0,0 +1,305 @@
+package docks
+
+import (
+	"fmt"
+	"os"
+	"runtime/pprof"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/safing/portmaster/spn/access"
+	"github.com/safing/portmaster/spn/cabin"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/ships"
+	"github.com/safing/portmaster/spn/terminal"
+)
+
+const defaultTestQueueSize = 200
+
+func TestExpansion(t *testing.T) {
+	t.Parallel()
+
+	// Test without and with encryption.
+	for _, encrypt := range []bool{false, true} {
+		// Test down/up separately and in parallel.
+		for _, parallel := range []bool{false, true} {
+			// Test with different flow controls.
+			for _, fc := range []struct {
+				flowControl     terminal.FlowControlType
+				flowControlSize uint32
+			}{
+				{
+					flowControl:     terminal.FlowControlNone,
+					flowControlSize: 5,
+				},
+				{
+					flowControl:     terminal.FlowControlDFQ,
+					flowControlSize: defaultTestQueueSize,
+				},
+			} {
+				// Run tests with combined options.
+				testExpansion(
+					t,
+					"expansion-hop-test",
+					&terminal.TerminalOpts{
+						Encrypt:         encrypt,
+						Padding:         8,
+						FlowControl:     fc.flowControl,
+						FlowControlSize: fc.flowControlSize,
+					},
+					defaultTestQueueSize,
+					defaultTestQueueSize,
+					parallel,
+				)
+			}
+		}
+	}
+
+	stressTestOpts := &terminal.TerminalOpts{
+		Encrypt:         true,
+		Padding:         8,
+		FlowControl:     terminal.FlowControlDFQ,
+		FlowControlSize: defaultTestQueueSize,
+	}
+	testExpansion(t, "expansion-stress-test-down", stressTestOpts, defaultTestQueueSize*100, 0, false)
+	testExpansion(t, "expansion-stress-test-up", stressTestOpts, 0, defaultTestQueueSize*100, false)
+	testExpansion(t, "expansion-stress-test-duplex", stressTestOpts, defaultTestQueueSize*100, defaultTestQueueSize*100, false)
+}
+
+func testExpansion( //nolint:maintidx,thelper
+	t *testing.T,
+	testID string,
+	terminalOpts *terminal.TerminalOpts,
+	clientCountTo,
+	serverCountTo uint64,
+	inParallel bool,
+) {
+	testID += fmt.Sprintf(":encrypt=%v,flowType=%d,parallel=%v", terminalOpts.Encrypt, terminalOpts.FlowControl, inParallel)
+
+	var identity2, identity3, identity4 *cabin.Identity
+	var connectedHub2, connectedHub3, connectedHub4 *hub.Hub
+	if terminalOpts.Encrypt {
+		identity2, connectedHub2 = getTestIdentity(t)
+		identity3, connectedHub3 = getTestIdentity(t)
+		identity4, connectedHub4 = getTestIdentity(t)
+	}
+
+	// Build ships and cranes.
+	optimalMinLoadSize = 100
+	ship1to2 := ships.NewTestShip(!terminalOpts.Encrypt, 100)
+	ship2to3 := ships.NewTestShip(!terminalOpts.Encrypt, 100)
+	ship3to4 := ships.NewTestShip(!terminalOpts.Encrypt, 100)
+
+	var crane1, crane2to1, crane2to3, crane3to2, crane3to4, crane4 *Crane
+	var craneWg sync.WaitGroup
+	craneWg.Add(6)
+
+	go func() {
+		var err error
+		crane1, err = NewCrane(ship1to2, connectedHub2, nil)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane1: %s", testID, err))
+		}
+		crane1.ID = "c1"
+		err = crane1.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane1: %s", testID, err))
+		}
+		crane1.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane2to1, err = NewCrane(ship1to2.Reverse(), nil, identity2)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane2to1: %s", testID, err))
+		}
+		crane2to1.ID = "c2to1"
+		err = crane2to1.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane2to1: %s", testID, err))
+		}
+		crane2to1.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane2to3, err = NewCrane(ship2to3, connectedHub3, nil)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane2to3: %s", testID, err))
+		}
+		crane2to3.ID = "c2to3"
+		err = crane2to3.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane2to3: %s", testID, err))
+		}
+		crane2to3.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane3to2, err = NewCrane(ship2to3.Reverse(), nil, identity3)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane3to2: %s", testID, err))
+		}
+		crane3to2.ID = "c3to2"
+		err = crane3to2.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane3to2: %s", testID, err))
+		}
+		crane3to2.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane3to4, err = NewCrane(ship3to4, connectedHub4, nil)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane3to4: %s", testID, err))
+		}
+		crane3to4.ID = "c3to4"
+		err = crane3to4.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane3to4: %s", testID, err))
+		}
+		crane3to4.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	go func() {
+		var err error
+		crane4, err = NewCrane(ship3to4.Reverse(), nil, identity4)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not create crane4: %s", testID, err))
+		}
+		crane4.ID = "c4"
+		err = crane4.Start(module.Ctx)
+		if err != nil {
+			panic(fmt.Sprintf("expansion test %s could not start crane4: %s", testID, err))
+		}
+		crane4.ship.MarkPublic()
+		craneWg.Done()
+	}()
+	craneWg.Wait()
+
+	// Assign cranes.
+	crane3HubID := testID + "-crane3HubID"
+	AssignCrane(crane3HubID, crane2to3)
+	crane4HubID := testID + "-crane4HubID"
+	AssignCrane(crane4HubID, crane3to4)
+
+	t.Logf("expansion test %s: initial setup complete", testID)
+
+	// Wait async for test to complete, print stack after timeout.
+	finished := make(chan struct{})
+	go func() {
+		select {
+		case <-finished:
+		case <-time.After(30 * time.Second):
+			fmt.Printf("expansion test %s is taking too long, print stack:\n", testID)
+			_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+			os.Exit(1)
+		}
+	}()
+
+	// Start initial crane.
+	homeTerminal, initData, tErr := NewLocalCraneTerminal(crane1, nil, &terminal.TerminalOpts{})
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to create home terminal: %s", testID, tErr)
+	}
+	tErr = crane1.EstablishNewTerminal(homeTerminal, initData)
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to connect home terminal: %s", testID, tErr)
+	}
+
+	t.Logf("expansion test %s: home terminal setup complete", testID)
+	time.Sleep(100 * time.Millisecond)
+
+	// Start counters for testing.
+	op0, tErr := terminal.NewCounterOp(homeTerminal, terminal.CounterOpts{
+		ClientCountTo: clientCountTo,
+		ServerCountTo: serverCountTo,
+	})
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to run counter op: %s", testID, tErr)
+	}
+	t.Logf("expansion test %s: home terminal counter setup complete", testID)
+	if !inParallel {
+		op0.Wait()
+	}
+
+	// Start expansion to crane 3.
+	opAuthTo2, tErr := access.AuthorizeToTerminal(homeTerminal)
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to auth with home terminal: %s", testID, tErr)
+	}
+	tErr = <-opAuthTo2.Result
+	if tErr.IsError() {
+		t.Fatalf("expansion test %s failed to auth with home terminal: %s", testID, tErr)
+	}
+	expansionTerminalTo3, err := ExpandTo(homeTerminal, crane3HubID, connectedHub3)
+	if err != nil {
+		t.Fatalf("expansion test %s failed to expand to %s: %s", testID, crane3HubID, tErr)
+	}
+
+	// Start counters for testing.
+	op1, tErr := terminal.NewCounterOp(expansionTerminalTo3, terminal.CounterOpts{
+		ClientCountTo: clientCountTo,
+		ServerCountTo: serverCountTo,
+	})
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to run counter op: %s", testID, tErr)
+	}
+
+	t.Logf("expansion test %s: expansion to crane3 and counter setup complete", testID)
+	if !inParallel {
+		op1.Wait()
+	}
+
+	// Start expansion to crane 4.
+	opAuthTo3, tErr := access.AuthorizeToTerminal(expansionTerminalTo3)
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to auth with extenstion terminal: %s", testID, tErr)
+	}
+	tErr = <-opAuthTo3.Result
+	if tErr.IsError() {
+		t.Fatalf("expansion test %s failed to auth with extenstion terminal: %s", testID, tErr)
+	}
+
+	expansionTerminalTo4, err := ExpandTo(expansionTerminalTo3, crane4HubID, connectedHub4)
+	if err != nil {
+		t.Fatalf("expansion test %s failed to expand to %s: %s", testID, crane4HubID, tErr)
+	}
+
+	// Start counters for testing.
+	op2, tErr := terminal.NewCounterOp(expansionTerminalTo4, terminal.CounterOpts{
+		ClientCountTo: clientCountTo,
+		ServerCountTo: serverCountTo,
+	})
+	if tErr != nil {
+		t.Fatalf("expansion test %s failed to run counter op: %s", testID, tErr)
+	}
+
+	t.Logf("expansion test %s: expansion to crane4 and counter setup complete", testID)
+	op2.Wait()
+
+	// Wait for op1 if not already.
+	if inParallel {
+		op0.Wait()
+		op1.Wait()
+	}
+
+	// Wait for completion.
+	close(finished)
+
+	// Wait a little so that all errors can be propagated, so we can truly see
+	// if we succeeded.
+	time.Sleep(100 * time.Millisecond)
+
+	// Check errors.
+	if op1.Error != nil {
+		t.Fatalf("crane test %s counter op1 failed: %s", testID, op1.Error)
+	}
+	if op2.Error != nil {
+		t.Fatalf("crane test %s counter op2 failed: %s", testID, op2.Error)
+	}
+}