Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

cmds/hub/build

@@ -56,5 +56,5 @@ echo "This information is useful for debugging and license compliance."
 echo "Run the compiled binary with the -version flag to see the information included."
 
 # build
-BUILD_PATH="github.com/safing/portbase/info"
+BUILD_PATH="github.com/safing/portmaster/base/info"
 go build $DEV -ldflags "-X ${BUILD_PATH}.commit=${BUILD_COMMIT} -X ${BUILD_PATH}.buildOptions=${BUILD_BUILDOPTIONS} -X ${BUILD_PATH}.buildUser=${BUILD_USER} -X ${BUILD_PATH}.buildHost=${BUILD_HOST} -X ${BUILD_PATH}.buildDate=${BUILD_DATE} -X ${BUILD_PATH}.buildSource=${BUILD_SOURCE}" $*

cmds/hub/main.go

@@ -1,20 +1,25 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"log/slog"
 	"os"
+	"os/signal"
 	"runtime"
+	"runtime/pprof"
+	"syscall"
+	"time"
 
-	"github.com/safing/portbase/info"
-	"github.com/safing/portbase/metrics"
-	"github.com/safing/portbase/modules"
-	"github.com/safing/portbase/run"
-	_ "github.com/safing/portmaster/service/core/base"
-	_ "github.com/safing/portmaster/service/ui"
+	"github.com/safing/portmaster/base/info"
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/base/metrics"
+	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/updates"
 	"github.com/safing/portmaster/service/updates/helper"
-	_ "github.com/safing/portmaster/spn/captain"
+	"github.com/safing/portmaster/spn"
 	"github.com/safing/portmaster/spn/conf"
 )
 
@@ -22,45 +27,131 @@ func init() {
 	flag.BoolVar(&updates.RebootOnRestart, "reboot-on-restart", false, "reboot server on auto-upgrade")
 }
 
+var sigUSR1 = syscall.Signal(0xa)
+
 func main() {
-	info.Set("SPN Hub", "0.7.7", "GPLv3")
+	flag.Parse()
+
+	// Set name and license.
+	info.Set("SPN Hub", "", "GPLv3")
 
 	// Configure metrics.
 	_ = metrics.SetNamespace("hub")
 
-	// Configure updating.
+	// Configure user agent and updates.
 	updates.UserAgent = fmt.Sprintf("SPN Hub (%s %s)", runtime.GOOS, runtime.GOARCH)
 	helper.IntelOnly()
 
-	// Configure SPN mode.
+	// Set SPN public hub mode.
 	conf.EnablePublicHub(true)
-	conf.EnableClient(false)
 
-	// Disable module management, as we want to start all modules.
-	modules.DisableModuleManagement()
+	// Set default log level.
+	log.SetLogLevel(log.WarningLevel)
+	_ = log.Start()
 
-	// Configure microtask threshold.
-	// Scale with CPU/GOMAXPROCS count, but keep a baseline and minimum:
-	// CPUs -> MicroTasks
-	//    0 ->  8 (increased to minimum)
-	//    1 ->  8 (increased to minimum)
-	//    2 ->  8
-	//    3 -> 10
-	//    4 -> 12
-	//    8 -> 20
-	//   16 -> 36
-	//
-	// Start with number of GOMAXPROCS.
-	microTasksThreshold := runtime.GOMAXPROCS(0) * 2
-	// Use at least 4 microtasks based on GOMAXPROCS.
-	if microTasksThreshold < 4 {
-		microTasksThreshold = 4
+	// Create instance.
+	var execCmdLine bool
+	instance, err := spn.New()
+	switch {
+	case err == nil:
+		// Continue
+	case errors.Is(err, mgr.ErrExecuteCmdLineOp):
+		execCmdLine = true
+	default:
+		fmt.Printf("error creating an instance: %s\n", err)
+		os.Exit(2)
 	}
-	// Add a 4 microtask baseline.
-	microTasksThreshold += 4
-	// Set threshold.
-	modules.SetMaxConcurrentMicroTasks(microTasksThreshold)
 
-	// Start.
-	os.Exit(run.Run())
+	// Execute command line operation, if requested or available.
+	switch {
+	case !execCmdLine:
+		// Run service.
+	case instance.CommandLineOperation == nil:
+		fmt.Println("command line operation execution requested, but not set")
+		os.Exit(3)
+	default:
+		// Run the function and exit.
+		err = instance.CommandLineOperation()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "command line operation failed: %s\n", err)
+			os.Exit(3)
+		}
+		os.Exit(0)
+	}
+
+	// Start
+	go func() {
+		err = instance.Start()
+		if err != nil {
+			fmt.Printf("instance start failed: %s\n", err)
+			os.Exit(1)
+		}
+	}()
+
+	// Wait for signal.
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(
+		signalCh,
+		os.Interrupt,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGQUIT,
+		sigUSR1,
+	)
+
+	select {
+	case sig := <-signalCh:
+		// Only print and continue to wait if SIGUSR1
+		if sig == sigUSR1 {
+			printStackTo(os.Stderr, "PRINTING STACK ON REQUEST")
+		} else {
+			fmt.Println(" <INTERRUPT>") // CLI output.
+			slog.Warn("program was interrupted, stopping")
+		}
+
+	case <-instance.Stopped():
+		log.Shutdown()
+		os.Exit(instance.ExitCode())
+	}
+
+	// Catch signals during shutdown.
+	// Rapid unplanned disassembly after 5 interrupts.
+	go func() {
+		forceCnt := 5
+		for {
+			<-signalCh
+			forceCnt--
+			if forceCnt > 0 {
+				fmt.Printf(" <INTERRUPT> again, but already shutting down - %d more to force\n", forceCnt)
+			} else {
+				printStackTo(os.Stderr, "PRINTING STACK ON FORCED EXIT")
+				os.Exit(1)
+			}
+		}
+	}()
+
+	// Rapid unplanned disassembly after 3 minutes.
+	go func() {
+		time.Sleep(3 * time.Minute)
+		printStackTo(os.Stderr, "PRINTING STACK - TAKING TOO LONG FOR SHUTDOWN")
+		os.Exit(1)
+	}()
+
+	// Stop instance.
+	if err := instance.Stop(); err != nil {
+		slog.Error("failed to stop", "err", err)
+	}
+	log.Shutdown()
+	os.Exit(instance.ExitCode())
+}
+
+func printStackTo(writer io.Writer, msg string) {
+	_, err := fmt.Fprintf(writer, "===== %s =====\n", msg)
+	if err == nil {
+		err = pprof.Lookup("goroutine").WriteTo(writer, 1)
+	}
+	if err != nil {
+		slog.Error("failed to write stack trace", "err", err)
+	}
 }