Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

spn/unit/scheduler.go

@@ -1,7 +1,6 @@
 package unit
 
 import (
-	"context"
 	"errors"
 	"math"
 	"sync"
@@ -9,6 +8,8 @@ import (
 	"time"
 
 	"github.com/tevino/abool"
+
+	"github.com/safing/portmaster/service/mgr"
 )
 
 const (
@@ -185,7 +186,7 @@ func (s *Scheduler) announceNextSlot() {
 
 // SlotScheduler manages the slot and schedules units.
 // Must only be started once.
-func (s *Scheduler) SlotScheduler(ctx context.Context) error {
+func (s *Scheduler) SlotScheduler(ctx *mgr.WorkerCtx) error {
 	// Start slot ticker.
 	ticker := time.NewTicker(s.config.SlotDuration / 2)
 	defer ticker.Stop()

spn/unit/scheduler_test.go

@@ -1,8 +1,9 @@
 package unit
 
 import (
-	"context"
 	"testing"
+
+	"github.com/safing/portmaster/service/mgr"
 )
 
 func BenchmarkScheduler(b *testing.B) {
@@ -10,21 +11,22 @@ func BenchmarkScheduler(b *testing.B) {
 
 	// Create and start scheduler.
 	s := NewScheduler(&SchedulerConfig{})
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
+	m := mgr.New("unit-test")
+	m.Go("test", func(ctx *mgr.WorkerCtx) error {
 		err := s.SlotScheduler(ctx)
 		if err != nil {
 			panic(err)
 		}
-	}()
-	defer cancel()
+		return nil
+	})
+	defer m.Cancel()
 
 	// Init control structures.
 	done := make(chan struct{})
 	finishedCh := make(chan struct{})
 
 	// Start workers.
-	for i := 0; i < workers; i++ {
+	for range workers {
 		go func() {
 			for {
 				u := s.NewUnit()
@@ -41,7 +43,7 @@ func BenchmarkScheduler(b *testing.B) {
 
 	// Start benchmark.
 	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
+	for range b.N {
 		<-finishedCh
 	}
 	b.StopTimer()

spn/unit/unit_debug.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/base/log"
 )
 
 // UnitDebugger is used to debug unit leaks.

spn/unit/unit_test.go

@@ -1,7 +1,6 @@
 package unit
 
 import (
-	"context"
 	"fmt"
 	"math"
 	"math/rand"
@@ -10,6 +9,8 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/safing/portmaster/service/mgr"
 )
 
 func TestUnit(t *testing.T) { //nolint:paralleltest
@@ -20,25 +21,27 @@ func TestUnit(t *testing.T) { //nolint:paralleltest
 	size := 1000000
 	workers := 100
 
+	m := mgr.New("unit-test")
 	// Create and start scheduler.
 	s := NewScheduler(&SchedulerConfig{})
 	s.StartDebugLog()
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		err := s.SlotScheduler(ctx)
+	// ctx, cancel := context.WithCancel(context.Background())
+	m.Go("test", func(w *mgr.WorkerCtx) error {
+		err := s.SlotScheduler(w)
 		if err != nil {
 			panic(err)
 		}
-	}()
-	defer cancel()
+		return nil
+	})
+	defer m.Cancel()
 
 	// Create 10 workers.
 	var wg sync.WaitGroup
 	wg.Add(workers)
 	sizePerWorker := size / workers
-	for i := 0; i < workers; i++ {
+	for range workers {
 		go func() {
-			for i := 0; i < sizePerWorker; i++ {
+			for range sizePerWorker {
 				u := s.NewUnit()
 
 				// Make 1% high priority.
@@ -96,7 +99,7 @@ func TestUnit(t *testing.T) { //nolint:paralleltest
 	)
 
 	// Shutdown
-	cancel()
+	m.Cancel()
 	time.Sleep(s.config.SlotDuration * 10)
 
 	// Check if scheduler shut down correctly.