Restructure modules (#1572)

* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
2024-08-09 17:15:48 +02:00
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
--- a/base/utils/call_limiter.go
+++ b/base/utils/call_limiter.go
@@ -0,0 +1,87 @@
+package utils
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// CallLimiter bundles concurrent calls and optionally limits how fast a function is called.
+type CallLimiter struct {
+	pause time.Duration
+
+	inLock   sync.Mutex
+	lastExec time.Time
+
+	waiters atomic.Int32
+	outLock sync.Mutex
+}
+
+// NewCallLimiter returns a new call limiter.
+// Set minPause to zero to disable the minimum pause between calls.
+func NewCallLimiter(minPause time.Duration) *CallLimiter {
+	return &CallLimiter{
+		pause: minPause,
+	}
+}
+
+// Do executes the given function.
+// All concurrent calls to Do are bundled and return when f() finishes.
+// Waits until the minimum pause is over before executing f() again.
+func (l *CallLimiter) Do(f func()) {
+	// Wait for the previous waiters to exit.
+	l.inLock.Lock()
+
+	// Defer final unlock to safeguard from panics.
+	defer func() {
+		// Execution is finished - leave.
+		// If we are the last waiter, let the next batch in.
+		if l.waiters.Add(-1) == 0 {
+			l.inLock.Unlock()
+		}
+	}()
+
+	// Check if we are the first waiter.
+	if l.waiters.Add(1) == 1 {
+		// Take the lead on this execution run.
+		l.lead(f)
+	} else {
+		// We are not the first waiter, let others in.
+		l.inLock.Unlock()
+	}
+
+	// Wait for execution to complete.
+	l.outLock.Lock()
+	l.outLock.Unlock() //nolint:staticcheck
+
+	// Last statement is in defer above.
+}
+
+func (l *CallLimiter) lead(f func()) {
+	// Make all others wait while we execute the function.
+	l.outLock.Lock()
+
+	// Unlock in lock until execution is finished.
+	l.inLock.Unlock()
+
+	// Transition from out lock to in lock when done.
+	defer func() {
+		// Update last execution time.
+		l.lastExec = time.Now().UTC()
+		// Stop newcomers from waiting on previous execution.
+		l.inLock.Lock()
+		// Allow waiters to leave.
+		l.outLock.Unlock()
+	}()
+
+	// Wait for the minimum duration between executions.
+	if l.pause > 0 {
+		sinceLastExec := time.Since(l.lastExec)
+		if sinceLastExec < l.pause {
+			time.Sleep(l.pause - sinceLastExec)
+		}
+	}
+
+	// Execute.
+	f()
+}