wip: migrate to mono-repo. SPN has already been moved to spn/

service/core/base/databases.go

@@ -0,0 +1,43 @@
+package base
+
+import (
+	"github.com/safing/portbase/database"
+	_ "github.com/safing/portbase/database/dbmodule"
+	_ "github.com/safing/portbase/database/storage/bbolt"
+)
+
+// Default Values (changeable for testing).
+var (
+	DefaultDatabaseStorageType = "bbolt"
+)
+
+func registerDatabases() error {
+	_, err := database.Register(&database.Database{
+		Name:        "core",
+		Description: "Holds core data, such as settings and profiles",
+		StorageType: DefaultDatabaseStorageType,
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = database.Register(&database.Database{
+		Name:        "cache",
+		Description: "Cached data, such as Intelligence and DNS Records",
+		StorageType: DefaultDatabaseStorageType,
+	})
+	if err != nil {
+		return err
+	}
+
+	// _, err = database.Register(&database.Database{
+	//   Name:        "history",
+	//   Description: "Historic event data",
+	//   StorageType: DefaultDatabaseStorageType,
+	// })
+	// if err != nil {
+	//   return err
+	// }
+
+	return nil
+}

service/core/base/global.go

@@ -0,0 +1,69 @@
+package base
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/dataroot"
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/modules"
+)
+
+// Default Values (changeable for testing).
+var (
+	DefaultAPIListenAddress = "127.0.0.1:817"
+
+	dataDir     string
+	databaseDir string
+	showVersion bool
+)
+
+func init() {
+	flag.StringVar(&dataDir, "data", "", "set data directory")
+	flag.StringVar(&databaseDir, "db", "", "alias to --data (deprecated)")
+	flag.BoolVar(&showVersion, "version", false, "show version and exit")
+
+	modules.SetGlobalPrepFn(globalPrep)
+}
+
+func globalPrep() error {
+	// check if meta info is ok
+	err := info.CheckVersion()
+	if err != nil {
+		return errors.New("compile error: please compile using the provided build script")
+	}
+
+	// print version
+	if showVersion {
+		fmt.Println(info.FullVersion())
+		return modules.ErrCleanExit
+	}
+
+	// check data root
+	if dataroot.Root() == nil {
+		// initialize data dir
+
+		// backwards compatibility
+		if dataDir == "" {
+			dataDir = databaseDir
+		}
+
+		// check data dir
+		if dataDir == "" {
+			return errors.New("please set the data directory using --data=/path/to/data/dir")
+		}
+
+		// initialize structure
+		err := dataroot.Initialize(dataDir, 0o0755)
+		if err != nil {
+			return err
+		}
+	}
+
+	// set api listen address
+	api.SetDefaultAPIListenAddress(DefaultAPIListenAddress)
+
+	return nil
+}

service/core/base/logs.go

@@ -0,0 +1,60 @@
+package base
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/safing/portbase/dataroot"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
+)
+
+const (
+	logTTL        = 30 * 24 * time.Hour
+	logFileDir    = "logs"
+	logFileSuffix = ".log"
+)
+
+func registerLogCleaner() {
+	module.NewTask("log cleaner", logCleaner).
+		Repeat(24 * time.Hour).
+		Schedule(time.Now().Add(15 * time.Minute))
+}
+
+func logCleaner(_ context.Context, _ *modules.Task) error {
+	ageThreshold := time.Now().Add(-logTTL)
+
+	return filepath.Walk(
+		filepath.Join(dataroot.Root().Path, logFileDir),
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				if !errors.Is(err, os.ErrNotExist) {
+					log.Warningf("core: failed to access %s while deleting old log files: %s", path, err)
+				}
+				return nil
+			}
+
+			switch {
+			case !info.Mode().IsRegular():
+				// Only delete regular files.
+			case !strings.HasSuffix(path, logFileSuffix):
+				// Only delete files that end with the correct suffix.
+			case info.ModTime().After(ageThreshold):
+				// Only delete files that are older that the log TTL.
+			default:
+				// Delete log file.
+				err := os.Remove(path)
+				if err != nil {
+					log.Warningf("core: failed to delete old log file %s: %s", path, err)
+				} else {
+					log.Tracef("core: deleted old log file %s", path)
+				}
+			}
+
+			return nil
+		})
+}

service/core/base/module.go

@@ -0,0 +1,38 @@
+package base
+
+import (
+	_ "github.com/safing/portbase/config"
+	_ "github.com/safing/portbase/metrics"
+	"github.com/safing/portbase/modules"
+	_ "github.com/safing/portbase/rng"
+)
+
+var module *modules.Module
+
+func init() {
+	module = modules.Register("base", nil, start, nil, "database", "config", "rng", "metrics")
+
+	// For prettier subsystem graph, printed with --print-subsystem-graph
+	/*
+		subsystems.Register(
+			"base",
+			"Base",
+			"THE GROUND.",
+			baseModule,
+			"",
+			nil,
+		)
+	*/
+}
+
+func start() error {
+	startProfiling()
+
+	if err := registerDatabases(); err != nil {
+		return err
+	}
+
+	registerLogCleaner()
+
+	return nil
+}

service/core/base/profiling.go

@@ -0,0 +1,41 @@
+package base
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"runtime/pprof"
+)
+
+var cpuProfile string
+
+func init() {
+	flag.StringVar(&cpuProfile, "cpuprofile", "", "write cpu profile to `file`")
+}
+
+func startProfiling() {
+	if cpuProfile != "" {
+		module.StartWorker("cpu profiler", cpuProfiler)
+	}
+}
+
+func cpuProfiler(ctx context.Context) error {
+	f, err := os.Create(cpuProfile)
+	if err != nil {
+		return fmt.Errorf("could not create CPU profile: %w", err)
+	}
+	if err := pprof.StartCPUProfile(f); err != nil {
+		return fmt.Errorf("could not start CPU profile: %w", err)
+	}
+
+	// wait for shutdown
+	<-ctx.Done()
+
+	pprof.StopCPUProfile()
+	err = f.Close()
+	if err != nil {
+		return fmt.Errorf("failed to close CPU profile file: %w", err)
+	}
+	return nil
+}