Merge pull request #1868 from safing/task/refactor-spn

Task: refactor spn to implement v2 indexes.

base/config/get_test.go

@@ -55,7 +55,7 @@ func TestGet(t *testing.T) { //nolint:paralleltest
 	// reset
 	options = make(map[string]*Option)
 
-	err := log.Start()
+	err := log.Start("info", true, "")
 	if err != nil {
 		t.Fatal(err)
 	}

cmds/hub/build

@@ -51,6 +51,12 @@ if [[ $1 == "dev" ]]; then
   DEV="-race"
 fi
 
+# Set GOOS and GOARCH for Darwin to use Linux architecture
+if [[ "$(uname)" == "Darwin" ]]; then
+  export GOOS=linux
+  export GOARCH=amd64
+fi
+
 echo "Please notice, that this build script includes metadata into the build."
 echo "This information is useful for debugging and license compliance."
 echo "Run the compiled binary with the -version flag to see the information included."

cmds/hub/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/safing/portmaster/service"
 	"github.com/safing/portmaster/service/configure"
 	"github.com/safing/portmaster/service/updates"
+	"github.com/safing/portmaster/spn"
 	"github.com/safing/portmaster/spn/conf"
 )
 
@@ -73,9 +74,31 @@ func initializeGlobals(cmd *cobra.Command, args []string) {
 	// Set SPN public hub mode.
 	conf.EnablePublicHub(true)
 
+	// Configure SPN binary updates.
+	configure.DefaultBinaryIndexName = "SPN Binaries"
+	configure.DefaultStableBinaryIndexURLs = []string{
+		"https://updates.safing.io/spn-stable.v3.json",
+	}
+	configure.DefaultBetaBinaryIndexURLs = []string{
+		"https://updates.safing.io/spn-beta.v3.json",
+	}
+	configure.DefaultStagingBinaryIndexURLs = []string{
+		"https://updates.safing.io/spn-staging.v3.json",
+	}
+	configure.DefaultSupportBinaryIndexURLs = []string{
+		"https://updates.safing.io/spn-support.v3.json",
+	}
+
+	if binDir == "" {
+		binDir = "/opt/safing/spn"
+	}
+	if dataDir == "" {
+		dataDir = "/opt/safing/spn"
+	}
+
 	// Configure service.
 	cmdbase.SvcFactory = func(svcCfg *service.ServiceConfig) (cmdbase.ServiceInstance, error) {
-		svc, err := service.New(svcCfg)
+		svc, err := spn.New(svcCfg)
 		return svc, err
 	}
 

cmds/winkext-test/main.go

@@ -45,7 +45,7 @@ func main() {
 	}
 
 	// logging
-	err := log.Start()
+	err := log.Start("info", true, "")
 	if err != nil {
 		fmt.Printf("failed to start logging: %s\n", err)
 		os.Exit(1)

service/configure/updates.go

@@ -6,7 +6,7 @@ import (
 
 var (
 	DefaultBinaryIndexName = "Portmaster Binaries"
-	DefaultIntelIndexName  = "intel"
+	DefaultIntelIndexName  = "Portmaster Intel"
 
 	DefaultStableBinaryIndexURLs = []string{
 		"https://updates.safing.io/stable.v3.json",

service/updates/index.go

@@ -241,7 +241,10 @@ func (index *Index) ShouldUpgradeTo(newIndex *Index) error {
 		return nil
 
 	case index.Name != newIndex.Name:
-		return errors.New("new index name does not match")
+		return fmt.Errorf(
+			"new index name (%q) does not match current index name (%q)",
+			newIndex.Name, index.Name,
+		)
 
 	case index.isLocallyGenerated:
 		if newIndex.versionNum.GreaterThan(index.versionNum) {

service/updates/module.go

@@ -18,7 +18,6 @@ import (
 	"github.com/safing/portmaster/base/log"
 	"github.com/safing/portmaster/base/notifications"
 	"github.com/safing/portmaster/base/utils"
-	"github.com/safing/portmaster/service/configure"
 	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/ui"
 )
@@ -222,7 +221,7 @@ func New(instance instance, name string, cfg Config) (*Updater, error) {
 		module.corruptedInstallation = fmt.Errorf("invalid index: %w", err)
 	}
 	index, err = GenerateIndexFromDir(cfg.Directory, IndexScanConfig{
-		Name:    configure.DefaultBinaryIndexName,
+		Name:    cfg.Name,
 		Version: info.VersionNumber(),
 	})
 	if err == nil && index.init(currentPlatform) == nil {

spn/docks/module_test.go

@@ -47,7 +47,7 @@ func (stub *testInstance) Stopping() bool {
 func (stub *testInstance) SetCmdLineOperation(f func() error) {}
 
 func runTest(m *testing.M) error {
-	_ = log.Start()
+	_ = log.Start("info", true, "")
 
 	ds, err := config.InitializeUnitTestDataroot("test-docks")
 	if err != nil {

spn/instance.go

@@ -21,6 +21,7 @@ import (
 	"github.com/safing/portmaster/service/intel/geoip"
 	"github.com/safing/portmaster/service/mgr"
 	"github.com/safing/portmaster/service/netenv"
+	"github.com/safing/portmaster/service/ui"
 	"github.com/safing/portmaster/service/updates"
 	"github.com/safing/portmaster/spn/access"
 	"github.com/safing/portmaster/spn/cabin"
@@ -74,6 +75,7 @@ type Instance struct {
 	ships     *ships.Ships
 	sluice    *sluice.SluiceModule
 	terminal  *terminal.TerminalModule
+	ui        *ui.UI
 
 	CommandLineOperation func() error
 	ShouldRestart        bool
@@ -136,6 +138,11 @@ func New(svcCfg *service.ServiceConfig) (*Instance, error) {
 	if err != nil {
 		return instance, fmt.Errorf("create updates config: %w", err)
 	}
+
+	//Enable autodownload and autoapply
+	binaryUpdateConfig.AutoDownload = true
+	binaryUpdateConfig.AutoApply = true
+
 	instance.binaryUpdates, err = updates.New(instance, "Binary Updater", *binaryUpdateConfig)
 	if err != nil {
 		return instance, fmt.Errorf("create updates module: %w", err)
@@ -371,6 +378,11 @@ func (i *Instance) Terminal() *terminal.TerminalModule {
 	return i.terminal
 }
 
+// UI returns the ui module.
+func (i *Instance) UI() *ui.UI {
+	return i.ui
+}
+
 // FilterLists returns the filterLists module.
 func (i *Instance) FilterLists() *filterlists.FilterLists {
 	return i.filterLists
@@ -508,6 +520,21 @@ func (i *Instance) ExitCode() int {
 	return int(i.exitCode.Load())
 }
 
+// ShouldRestartIsSet returns whether the service/instance should be restarted.
+func (i *Instance) ShouldRestartIsSet() bool {
+	return i.ShouldRestart
+}
+
+// CommandLineOperationIsSet returns whether the command line option is set.
+func (i *Instance) CommandLineOperationIsSet() bool {
+	return i.CommandLineOperation != nil
+}
+
+// CommandLineOperationExecute executes the set command line option.
+func (i *Instance) CommandLineOperationExecute() error {
+	return i.CommandLineOperation()
+}
+
 // SPNGroup fakes interface conformance.
 // SPNGroup is only needed on SPN clients.
 func (i *Instance) SPNGroup() *mgr.ExtendedGroup {

spn/testing/simple/entrypoint.sh

@@ -14,4 +14,4 @@ cat /opt/shared/config-template.json | sed "s/\$HUBNAME/$HUBNAME/g" > /opt/data/
 BIN=$(ls /opt/ | grep hub)
 
 # Start Hub.
-/opt/$BIN --data /opt/data --log trace --spn-map test --bootstrap-file /opt/shared/bootstrap.dsd --api-address 0.0.0.0:817 --devmode
+/opt/$BIN  --log trace --spn-map test --bootstrap-file /opt/shared/bootstrap.dsd --api-address 0.0.0.0:817 --devmode

spn/tools/Dockerfile

@@ -3,18 +3,18 @@ FROM alpine as builder
 # Ensure ca-certficates are up to date
 # RUN update-ca-certificates
 
-# Download and verify portmaster-start binary.
+# Download and verify spn-hub binary.
 RUN mkdir /init
-RUN wget https://updates.safing.io/linux_amd64/start/portmaster-start_v0-9-6 -O /init/portmaster-start 
+RUN wget https://updates.safing.io/latest/linux_amd64/hub/spn-hub -O /init/spn-hub
 COPY start-checksum.txt /init/start-checksum
 RUN cd /init && sha256sum -c /init/start-checksum
-RUN chmod 555 /init/portmaster-start
+RUN chmod 555 /init/spn-hub
 
 # Use minimal image as base.
 FROM alpine
 
 # Copy the static executable.
-COPY --from=builder /init/portmaster-start /init/portmaster-start 
+COPY --from=builder /init/spn-hub /init/spn-hub
 
 # Copy the init script
 COPY container-init.sh /init.sh

spn/tools/container-init.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 
 DATA="/data"
-START="/data/portmaster-start"
+START="/data/spn-hub"
-INIT_START="/init/portmaster-start"
+INIT_START="/init/spn-hub"
 
 # Set safe shell options.
 set -euf -o pipefail
@@ -18,13 +18,10 @@ if [ ! -f $START ]; then
   cp $INIT_START $START
 fi
 
-# Download updates.
-echo "running: $START update --data /data --intel-only"
-$START update --data /data --intel-only
 
 # Remove PID file, which could have been left after a crash.
 rm -f $DATA/hub-lock.pid
 
 # Always start the SPN Hub with the updated main start binary.
-echo "running: $START hub --data /data -- $@"
+echo "running: $START"
-$START hub --data /data -- $@
+$START -- $@

spn/tools/install.sh

@@ -323,4 +323,4 @@ EOT
     setup_systemd
 }
 
-main "$@"
+main "$@"

spn/tools/install.v2.sh

@@ -0,0 +1,322 @@
+#!/bin/sh
+#
+# This script should be run via curl as root:
+#   sudo sh -c "$(curl -fsSL https://raw.githubusercontent.com/safing/portmaster/master/spn/tools/install-spn.sh)"
+# or wget
+#   sudo sh -c "$(wget -qO- https://raw.githubusercontent.com/safing/portmaster/master/spn/tools/install-spn.sh)"
+#
+# As an alternative, you can first download the install script and run it afterwards:
+#   wget https://raw.githubusercontent.com/safing/portmaster/master/spn/tools/install-spn.sh
+#   sudo sh ./install.sh
+#
+#
+set -e
+
+ARCH=
+INSTALLDIR=
+SPNBINARY=
+ENABLENOW=
+INSTALLSYSTEMD=
+SYSTEMDINSTALLPATH=
+LOGDIR=
+
+apply_defaults() {
+    ARCH=${ARCH:-amd64}
+    INSTALLDIR=${INSTALLDIR:-/opt/safing/spn}
+    SPNBINARY=${SPNBINARY:-https://updates.safing.io/latest/linux_${ARCH}/hub/spn-hub}
+    SYSTEMDINSTALLPATH=${SYSTEMDINSTALLPATH:-/etc/systemd/system/spn.service}
+    LOGDIR=${LOGDIR:-/opt/safing/spn}
+
+    if command_exists systemctl; then
+        INSTALLSYSTEMD=${INSTALLSYSTEMD:-yes}
+        ENABLENOW=${ENABLENOW:-yes}
+    else
+        INSTALLSYSTEMD=${INSTALLSYSTEMD:-no}
+        ENABLENOW=${ENABLENOW:-no}
+    fi
+
+    # The hostname may be freshly set, ensure the ENV variable is correct.
+    export HOSTNAME=$(hostname)
+}
+
+command_exists() {
+    command -v "$@" >/dev/null 2>&1
+}
+
+setup_tty() {
+    if [ -t 0 ]; then
+        interactive=yes
+    fi
+
+	if [ -t 1 ]; then
+		RED=$(printf '\033[31m')
+		GREEN=$(printf '\033[32m')
+		YELLOW=$(printf '\033[33m')
+		BLUE=$(printf '\033[34m')
+		BOLD=$(printf '\033[1m')
+		RESET=$(printf '\033[m')
+	else
+		RED=""
+		GREEN=""
+		YELLOW=""
+		BLUE=""
+		BOLD=""
+		RESET=""
+	fi
+}
+
+log() {
+    echo ${GREEN}${BOLD}"-> "${RESET}"$@" >&2
+}
+
+error() {
+    echo ${RED}"Error: $@"${RESET} >&2
+}
+
+warn() {
+    echo ${YELLOW}"warn: $@"${RESET} >&2
+}
+
+run_systemctl() {
+    systemctl $@ >/dev/null 2>&1 
+}
+
+download_file() {
+    local src=$1
+    local dest=$2
+
+    if command_exists curl; then
+        curl --silent --fail --show-error --location --output $dest $src
+    elif command_exists wget; then
+        wget --quiet -O $dest $src
+    else
+        error "No suitable download command found, either curl or wget must be installed"
+        exit 1
+    fi
+}
+
+ensure_install_dir() {
+    log "Creating ${INSTALLDIR}"
+    mkdir -p ${INSTALLDIR}
+}
+
+download_spnbinary() {
+    log "Downloading SPN binary ..."
+    local dest="${INSTALLDIR}/hub"
+    if [ -f "${dest}" ]; then
+        warn "Overwriting existing hub at ${dest}"
+    fi
+
+    download_file ${SPNBINARY} ${dest}
+
+    log "Changing permissions"
+    chmod a+x ${dest}
+}
+
+setup_systemd() {
+    log "Installing systemd service unit ..."
+    if [ ! "${INSTALLSYSTEMD}" = "yes" ]; then
+        warn "Skipping setup of systemd service unit"
+        echo "To launch the hub, execute the following as root:"
+        echo ""
+        echo "${INSTALLDIR}/hub --data-dir ${INSTALLDIR}"
+        echo ""
+        return
+    fi
+
+    if [ -f "${SYSTEMDINSTALLPATH}" ]; then
+        warn "Overwriting existing unit path"
+    fi
+
+    cat >${SYSTEMDINSTALLPATH} <<EOT
+[Unit]
+Description=Safing Privacy Network Hub
+Wants=nss-lookup.target
+Conflicts=shutdown.target
+Before=shutdown.target
+
+[Service]
+Type=simple
+Restart=on-failure
+RestartSec=5
+LimitNOFILE=infinity
+Environment=LOGLEVEL=warning
+Environment=SPN_ARGS=
+EnvironmentFile=-/etc/default/spn
+ExecStart=${INSTALLDIR}/hub --data-dir ${INSTALLDIR} --log \$LOGLEVEL --log-dir ${LOGDIR} \$SPN_ARGS
+
+[Install]
+WantedBy=multi-user.target
+EOT
+
+    log "Reloading systemd unit files"
+    run_systemctl daemon-reload
+
+    if run_systemctl is-active spn ||
+       run_systemctl is-failed spn; then
+        log "Restarting SPN hub"
+        run_systemctl restart spn.service
+    fi
+
+    # TODO(ppacher): allow disabling enable
+    if ! run_systemctl is-enabled spn ; then
+        if [ "${ENABLENOW}" = "yes" ]; then
+            log "Enabling and starting SPN."
+            run_systemctl enable --now spn.service || exit 1
+
+            log "Watch logs using: journalctl -fu spn.service"
+        else
+            log "Enabling SPN"
+            run_systemctl enable spn.service || exit 1
+        fi
+    fi
+
+}
+
+ask_config() {
+    if [ "${HOSTNAME}" = "" ]; then
+        log "Please enter hostname:"
+        read -p "> " HOSTNAME
+    fi
+    if [ "${METRICS_COMMENT}" = "" ]; then
+        log "Please enter metrics comment:"
+        read -p "> " METRICS_COMMENT
+    fi
+}
+
+write_config_file() {
+    cat >${1} <<EOT
+{
+  "core": {
+    "metrics": {
+      "instance": "$HOSTNAME",
+      "comment": "$METRICS_COMMENT",
+      "push": "$PUSHMETRICS"
+    }
+  },
+  "spn": {
+    "publicHub": {
+      "name": "$HOSTNAME"
+    }
+  }
+}
+EOT
+}
+
+confirm_config() {
+    log "Installation configuration:"
+    echo ""
+    echo "   Architecture: ${BOLD}${ARCH}${RESET}"
+    echo "   Download-URL: ${BOLD}${SPNBINARY}${RESET}"
+    echo "     Target Dir: ${BOLD}${INSTALLDIR}${RESET}"
+    echo "Install systemd: ${BOLD}${INSTALLSYSTEMD}${RESET}"
+    echo "      Unit path: ${BOLD}${SYSTEMDINSTALLPATH}${RESET}"
+    echo "      Start Now: ${BOLD}${ENABLENOW}${RESET}"
+    echo ""
+    echo "         Config:"
+    tmpfile=$(mktemp)
+    write_config_file $tmpfile
+    cat $tmpfile
+    echo ""
+    echo ""
+
+    if [ ! -z "${interactive}" ]
+    then
+        read -p "Continue (Y/n)? " ans
+        case "$ans" in
+            "" | "y" | "Y")
+                echo ""
+                ;;
+            **)
+                error "User aborted"
+                exit 1
+        esac
+    fi
+}
+
+print_help() {
+    cat <<EOT
+Usage: $0 [OPTIONS...]
+
+${BOLD}Options:${RESET}
+    ${GREEN}-y, --unattended${RESET}           Don't ask for confirmation.
+    ${GREEN}-n, --no-start${RESET}             Do not immediately start SPN hub.
+    ${GREEN}-t, --target PATH${RESET}          Configure the installation directory.
+    ${GREEN}-h, --help${RESET}                 Display this help text
+    ${GREEN}-a, --arch${RESET}                 Configure the binary architecture.
+    ${GREEN}-u, --url URL${RESET}              Set download URL for spn-hub.
+    ${GREEN}-S, --no-systemd${RESET}           Do not install systemd service unit.
+    ${GREEN}-s, --service-path PATH${RESET}    Location for the systemd unit file.
+EOT
+}
+
+main() {
+    setup_tty
+
+    # Parse arguments
+    while [ $# -gt 0 ]
+    do
+        case $1 in
+            --unattended | -y)
+                interactive=""
+                ;;
+            --no-start | -n)
+                ENABLENOW="no"
+                ;;
+            --target | -t)
+                INSTALLDIR=$2
+                shift
+                ;;
+            --help | -h)
+                print_help
+                exit 1 ;;
+            --arch | -a)
+                ARCH=$2
+                shift
+                ;;
+            --url | -u)
+                SPNBINARY=$2
+                shift
+                ;;
+            --no-systemd | -S)
+                INSTALLSYSTEMD=no
+                ENABLENOW=no
+                ;;
+            --service-path | -s)
+                SYSTEMDINSTALLPATH=$2
+                shift
+                ;;
+            *)
+                error "Unknown flag $1"
+                exit 1
+                ;;
+        esac
+        shift
+    done
+
+    cat <<EOT
+${BLUE}${BOLD}
+          ▄▄▄▄  ▄▄▄▄▄  ▄▄   ▄
+         █▀   ▀ █   ▀█ █▀▄  █
+         ▀█▄▄▄  █▄▄▄█▀ █ █▄ █
+             ▀█ █      █  █ █
+         ▀▄▄▄█▀ █      █   ██
+        ${GREEN}Safing Privacy Network
+${RESET}
+EOT
+
+    # prepare config
+    apply_defaults
+    ask_config
+    confirm_config
+
+    # Setup hub
+    ensure_install_dir
+    download_spnbinary
+    write_config_file "${INSTALLDIR}/config.json"
+
+    # setup systemd
+    setup_systemd
+}
+
+main "$@"

spn/tools/start-checksum.txt

@@ -1 +1 @@
-3f45f0814c6db28c3899b39ae0ab01f8f20a8cc98697dbe8039162ccd9590bf8  ./portmaster-start
+da0ca5ca57f3f5e80a7cb61a8e0ad9b1423051fc12e518b0539c7c69b7a68ee8  ./spn-hub