Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>

base/utils/debug/debug.go

@@ -0,0 +1,131 @@
+package debug
+
+import (
+	"bytes"
+	"fmt"
+	"runtime/pprof"
+	"time"
+
+	"github.com/safing/portmaster/base/info"
+	"github.com/safing/portmaster/base/log"
+)
+
+// Info gathers debugging information and stores everything in a buffer in
+// order to write it to somewhere later. It directly inherits a bytes.Buffer,
+// so you can also use all these functions too.
+type Info struct {
+	bytes.Buffer
+	Style string
+}
+
+// InfoFlag defines possible options for adding sections to a Info.
+type InfoFlag int
+
+const (
+	// NoFlags does nothing.
+	NoFlags InfoFlag = 0
+
+	// UseCodeSection wraps the section content in a markdown code section.
+	UseCodeSection InfoFlag = 1
+
+	// AddContentLineBreaks adds a line breaks after each line of content,
+	// except for the last.
+	AddContentLineBreaks InfoFlag = 2
+)
+
+func useCodeSection(flags InfoFlag) bool {
+	return flags&UseCodeSection > 0
+}
+
+func addContentLineBreaks(flags InfoFlag) bool {
+	return flags&AddContentLineBreaks > 0
+}
+
+// AddSection adds a debug section to the Info. The result is directly
+// written into the buffer.
+func (di *Info) AddSection(name string, flags InfoFlag, content ...string) {
+	// Check if we need a spacer.
+	if di.Len() > 0 {
+		_, _ = di.WriteString("\n\n")
+	}
+
+	// Write section to buffer.
+
+	// Write section header.
+	if di.Style == "github" {
+		_, _ = di.WriteString(fmt.Sprintf("<details>\n<summary>%s</summary>\n\n", name))
+	} else {
+		_, _ = di.WriteString(fmt.Sprintf("**%s**:\n\n", name))
+	}
+
+	// Write section content.
+	if useCodeSection(flags) {
+		// Write code header: Needs one empty line between previous data.
+		_, _ = di.WriteString("```\n")
+	}
+	for i, part := range content {
+		_, _ = di.WriteString(part)
+		if addContentLineBreaks(flags) && i < len(content)-1 {
+			_, _ = di.WriteString("\n")
+		}
+	}
+	if useCodeSection(flags) {
+		// Write code footer: Needs one empty line between next data.
+		_, _ = di.WriteString("\n```\n")
+	}
+
+	// Write section header.
+	if di.Style == "github" {
+		_, _ = di.WriteString("\n</details>")
+	}
+}
+
+// AddVersionInfo adds version information from the info pkg.
+func (di *Info) AddVersionInfo() {
+	di.AddSection(
+		"Version "+info.Version(),
+		UseCodeSection,
+		info.FullVersion(),
+	)
+}
+
+// AddGoroutineStack adds the current goroutine stack.
+func (di *Info) AddGoroutineStack() {
+	buf := new(bytes.Buffer)
+	err := pprof.Lookup("goroutine").WriteTo(buf, 1)
+	if err != nil {
+		di.AddSection(
+			"Goroutine Stack",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Add section.
+	di.AddSection(
+		"Goroutine Stack",
+		UseCodeSection,
+		buf.String(),
+	)
+}
+
+// AddLastUnexpectedLogs adds the last 10 unexpected log lines, if any.
+func (di *Info) AddLastUnexpectedLogs() {
+	lines := log.GetLastUnexpectedLogs()
+
+	// Check if there is anything at all.
+	if len(lines) == 0 {
+		di.AddSection("No Unexpected Logs", NoFlags)
+		return
+	}
+
+	di.AddSection(
+		"Unexpected Logs",
+		UseCodeSection|AddContentLineBreaks,
+		append(
+			lines,
+			fmt.Sprintf("%s CURRENT TIME", time.Now().Format("060102 15:04:05.000")),
+		)...,
+	)
+}

base/utils/debug/debug_android.go

@@ -0,0 +1,31 @@
+package debug
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/safing/portmaster-android/go/app_interface"
+)
+
+// AddPlatformInfo adds OS and platform information.
+func (di *Info) AddPlatformInfo(_ context.Context) {
+	// Get information from the system.
+	info, err := app_interface.GetPlatformInfo()
+	if err != nil {
+		di.AddSection(
+			"Platform Information",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Add section.
+	di.AddSection(
+		fmt.Sprintf("Platform: Android"),
+		UseCodeSection|AddContentLineBreaks,
+		fmt.Sprintf("SDK: %d", info.SDK),
+		fmt.Sprintf("Device: %s %s (%s)", info.Manufacturer, info.Brand, info.Board),
+		fmt.Sprintf("App: %s: %s %s", info.ApplicationID, info.VersionName, info.BuildType))
+
+}

base/utils/debug/debug_default.go

@@ -0,0 +1,43 @@
+//go:build !android
+
+package debug
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/shirou/gopsutil/host"
+)
+
+// AddPlatformInfo adds OS and platform information.
+func (di *Info) AddPlatformInfo(ctx context.Context) {
+	// Get information from the system.
+	info, err := host.InfoWithContext(ctx)
+	if err != nil {
+		di.AddSection(
+			"Platform Information",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Check if we want to add virtulization information.
+	var virtInfo string
+	if info.VirtualizationRole == "guest" {
+		if info.VirtualizationSystem != "" {
+			virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem)
+		} else {
+			virtInfo = "VM: unidentified"
+		}
+	}
+
+	// Add section.
+	di.AddSection(
+		fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion),
+		UseCodeSection|AddContentLineBreaks,
+		fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion),
+		fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch),
+		virtInfo,
+	)
+}