Merge pull request #199 from safing/feature/improve-updates-and-releasing

Improve updates and releasing

cmds/portmaster-core/pack

@@ -3,14 +3,16 @@
 baseDir="$( cd "$(dirname "$0")" && pwd )"
 cd "$baseDir"
 
-COL_OFF="\033[00m"
+COL_OFF="\033[0m"
 COL_BOLD="\033[01;01m"
 COL_RED="\033[31m"
+COL_GREEN="\033[32m"
+COL_YELLOW="\033[33m"
 
 destDirPart1="../../dist"
 destDirPart2="core"
 
-function check {
+function prep {
   # output
   output="main"
   # get version
@@ -25,46 +27,47 @@ function check {
   fi
   # build destination path
   destPath=${destDirPart1}/${platform}/${destDirPart2}/$filename
+}
+
+function check {
+  prep
 
   # check if file exists
   if [[ -f $destPath ]]; then
-    echo "[core] $platform $version already built"
+    echo "[core] $platform v$version already built"
   else
-    echo -e "${COL_BOLD}[core] $platform $version${COL_OFF}"
+    echo -e "${COL_BOLD}[core] $platform v$version${COL_OFF}"
   fi
 }
 
 function build {
-  # output
+  prep
-  output="main"
-  # get version
-  version=$(grep "info.Set" main.go | cut -d'"' -f4)
-  # build versioned file name
-  filename="portmaster-core_v${version//./-}"
-  # platform
-  platform="${GOOS}_${GOARCH}"
-  if [[ $GOOS == "windows" ]]; then
-    filename="${filename}.exe"
-    output="${output}.exe"
-  fi
-  # build destination path
-  destPath=${destDirPart1}/${platform}/${destDirPart2}/$filename
 
   # check if file exists
   if [[ -f $destPath ]]; then
-    echo "[core] $platform already built in version $version, skipping..."
+    echo "[core] $platform already built in v$version, skipping..."
     return
   fi
 
   # build
   ./build main.go
   if [[ $? -ne 0 ]]; then
-    echo -e "\n${COL_BOLD}[core] $platform: ${COL_RED}BUILD FAILED.${COL_OFF}"
+    echo -e "\n${COL_BOLD}[core] $platform v$version: ${COL_RED}BUILD FAILED.${COL_OFF}"
     exit 1
   fi
   mkdir -p $(dirname $destPath)
   cp $output $destPath
-  echo -e "\n${COL_BOLD}[core] $platform: successfully built.${COL_OFF}"
+  echo -e "\n${COL_BOLD}[core] $platform v$version: ${COL_GREEN}successfully built.${COL_OFF}"
+}
+
+function reset {
+  prep
+
+  # delete if file exists
+  if [[ -f $destPath ]]; then
+    rm $destPath
+    echo "[core] $platform v$version deleted."
+  fi
 }
 
 function check_all {
@@ -79,6 +82,12 @@ function build_all {
   GOOS=darwin GOARCH=amd64 build
 }
 
+function reset_all {
+  GOOS=linux GOARCH=amd64 reset
+  GOOS=windows GOARCH=amd64 reset
+  GOOS=darwin GOARCH=amd64 reset
+}
+
 case $1 in
   "check" )
     check_all
@@ -86,6 +95,9 @@ case $1 in
   "build" )
     build_all
     ;;
+  "reset" )
+    reset_all
+    ;;
   * )
     echo ""
     echo "build list:"

cmds/portmaster-start/main.go

@@ -22,6 +22,7 @@ import (
 
 var (
 	dataDir    string
+	staging    bool
 	maxRetries int
 	dataRoot   *utils.DirStructure
 	logsRoot   *utils.DirStructure
@@ -41,8 +42,8 @@ var (
 		Use:   "portmaster-start",
 		Short: "Start Portmaster components",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
-			mustLoadIndex := cmd == updatesCmd
+			mustLoadIndex := indexRequired(cmd)
-			if err := configureDataRoot(mustLoadIndex); err != nil {
+			if err := configureRegistry(mustLoadIndex); err != nil {
 				return err
 			}
 
@@ -64,8 +65,9 @@ func init() {
 	{
 		flags.StringVar(&dataDir, "data", "", "Configures the data directory. Alternatively, this can also be set via the environment variable PORTMASTER_DATA.")
 		flags.StringVar(&registry.UserAgent, "update-agent", "Start", "Sets the user agent for requests to the update server")
+		flags.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)")
 		flags.IntVar(&maxRetries, "max-retries", 5, "Maximum number of retries when starting a Portmaster component")
-		flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdid.")
+		flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdin.")
 		_ = rootCmd.MarkPersistentFlagDirname("data")
 		_ = flags.MarkHidden("input-signals")
 	}
@@ -131,34 +133,32 @@ func initCobra() {
 	portlog.SetLogLevel(portlog.CriticalLevel)
 }
 
-func configureDataRoot(mustLoadIndex bool) error {
+func configureRegistry(mustLoadIndex bool) error {
-	// The data directory is not
+	// If dataDir is not set, check the environment variable.
-	// check for environment variable
-	// PORTMASTER_DATA
 	if dataDir == "" {
 		dataDir = os.Getenv("PORTMASTER_DATA")
 	}
 
-	// if it's still empty try to auto-detect it
+	// If it's still empty, try to auto-detect it.
 	if dataDir == "" {
 		dataDir = detectInstallationDir()
 	}
 
-	// finally, if it's still empty the user must provide it
+	// Finally, if it's still empty, the user must provide it.
 	if dataDir == "" {
 		return errors.New("please set the data directory using --data=/path/to/data/dir")
 	}
 
-	// remove redundant escape characters and quotes
+	// Remove left over quotes.
 	dataDir = strings.Trim(dataDir, `\"`)
-	// initialize dataroot
+	// Initialize data root.
 	err := dataroot.Initialize(dataDir, 0755)
 	if err != nil {
 		return fmt.Errorf("failed to initialize data root: %s", err)
 	}
 	dataRoot = dataroot.Root()
 
-	// initialize registry
+	// Initialize registry.
 	err = registry.Initialize(dataRoot.ChildDir("updates", 0755))
 	if err != nil {
 		return err
@@ -177,6 +177,19 @@ func configureDataRoot(mustLoadIndex bool) error {
 	// Beta:   true,
 	// })
 
+	if stagingActive() {
+		// Set flag no matter how staging was activated.
+		staging = true
+
+		log.Println("WARNING: staging environment is active.")
+
+		registry.AddIndex(updater.Index{
+			Path:   "staging.json",
+			Stable: true,
+			Beta:   true,
+		})
+	}
+
 	return updateRegistryIndex(mustLoadIndex)
 }
 
@@ -233,3 +246,14 @@ func detectInstallationDir() string {
 
 	return parent
 }
+
+func stagingActive() bool {
+	// Check flag and env variable.
+	if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" {
+		return true
+	}
+
+	// Check if staging index is present and acessible.
+	_, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json"))
+	return err == nil
+}

cmds/portmaster-start/pack

@@ -3,14 +3,16 @@
 baseDir="$( cd "$(dirname "$0")" && pwd )"
 cd "$baseDir"
 
-COL_OFF="\033[00m"
+COL_OFF="\033[0m"
 COL_BOLD="\033[01;01m"
 COL_RED="\033[31m"
+COL_GREEN="\033[32m"
+COL_YELLOW="\033[33m"
 
 destDirPart1="../../dist"
 destDirPart2="start"
 
-function check {
+function prep {
   # output
   output="portmaster-start"
   # get version
@@ -25,46 +27,47 @@ function check {
   fi
   # build destination path
   destPath=${destDirPart1}/${platform}/${destDirPart2}/$filename
+}
+
+function check {
+  prep
 
   # check if file exists
   if [[ -f $destPath ]]; then
     echo "[start] $platform $version already built"
   else
-    echo -e "${COL_BOLD}[start] $platform $version${COL_OFF}"
+    echo -e "${COL_BOLD}[start] $platform v$version${COL_OFF}"
   fi
 }
 
 function build {
-  # output
+  prep
-  output="portmaster-start"
-  # get version
-  version=$(grep "info.Set" main.go | cut -d'"' -f4)
-  # build versioned file name
-  filename="portmaster-start_v${version//./-}"
-  # platform
-  platform="${GOOS}_${GOARCH}"
-  if [[ $GOOS == "windows" ]]; then
-    filename="${filename}.exe"
-    output="${output}.exe"
-  fi
-  # build destination path
-  destPath=${destDirPart1}/${platform}/${destDirPart2}/$filename
 
   # check if file exists
   if [[ -f $destPath ]]; then
-    echo "[start] $platform already built in version $version, skipping..."
+    echo "[start] $platform already built in v$version, skipping..."
     return
   fi
 
   # build
   ./build
   if [[ $? -ne 0 ]]; then
-    echo -e "\n${COL_BOLD}[start] $platform: ${COL_RED}BUILD FAILED.${COL_OFF}"
+    echo -e "\n${COL_BOLD}[start] $platform v$version: ${COL_RED}BUILD FAILED.${COL_OFF}"
     exit 1
   fi
   mkdir -p $(dirname $destPath)
   cp $output $destPath
-  echo -e "\n${COL_BOLD}[start] $platform: successfully built.${COL_OFF}"
+  echo -e "\n${COL_BOLD}[start] $platform v$version: ${COL_GREEN}successfully built.${COL_OFF}"
+}
+
+function reset {
+  prep
+
+  # delete if file exists
+  if [[ -f $destPath ]]; then
+    rm $destPath
+    echo "[start] $platform v$version deleted."
+  fi
 }
 
 function check_all {
@@ -79,6 +82,12 @@ function build_all {
   GOOS=darwin GOARCH=amd64 build
 }
 
+function reset_all {
+  GOOS=linux GOARCH=amd64 reset
+  GOOS=windows GOARCH=amd64 reset
+  GOOS=darwin GOARCH=amd64 reset
+}
+
 case $1 in
   "check" )
     check_all
@@ -86,6 +95,9 @@ case $1 in
   "build" )
     build_all
     ;;
+  "reset" )
+    reset_all
+    ;;
   * )
     echo ""
     echo "build list:"

cmds/portmaster-start/run.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -19,6 +20,9 @@ const (
 	// RestartExitCode is the exit code that any service started by portmaster-start
 	// can return in order to trigger a restart after a clean shutdown.
 	RestartExitCode = 23
+
+	exeSuffix = ".exe"
+	zipSuffix = ".zip"
 )
 
 var (
@@ -49,7 +53,7 @@ func init() {
 		},
 		{
 			Name:              "Portmaster App",
-			Identifier:        "app/portmaster-app",
+			Identifier:        "app/portmaster-app.zip",
 			AllowDownload:     false,
 			AllowHidingWindow: false,
 		},
@@ -62,7 +66,6 @@ func init() {
 		{
 			Name:              "Safing Privacy Network",
 			Identifier:        "hub/spn-hub",
-			ShortIdentifier:   "hub",
 			AllowDownload:     true,
 			AllowHidingWindow: true,
 		},
@@ -147,8 +150,8 @@ func run(opts *Options, cmdArgs []string) (err error) {
 	}()
 
 	// adapt identifier
-	if onWindows {
+	if onWindows && !strings.HasSuffix(opts.Identifier, zipSuffix) {
-		opts.Identifier += ".exe"
+		opts.Identifier += exeSuffix
 	}
 
 	// setup logging
@@ -275,16 +278,30 @@ func execute(opts *Options, args []string) (cont bool, err error) {
 	if err != nil {
 		return true, fmt.Errorf("could not get component: %w", err)
 	}
+	binPath := file.Path()
+
+	// Adapt path for packaged software.
+	if strings.HasSuffix(binPath, zipSuffix) {
+		// Remove suffix from binary path.
+		binPath = strings.TrimSuffix(binPath, zipSuffix)
+		// Add binary with the same name to access the unpacked binary.
+		binPath = filepath.Join(binPath, filepath.Base(binPath))
+
+		// Adapt binary path on Windows.
+		if onWindows {
+			binPath += exeSuffix
+		}
+	}
 
 	// check permission
-	if err := fixExecPerm(file.Path()); err != nil {
+	if err := fixExecPerm(binPath); err != nil {
 		return true, err
 	}
 
-	log.Printf("starting %s %s\n", file.Path(), strings.Join(args, " "))
+	log.Printf("starting %s %s\n", binPath, strings.Join(args, " "))
 
 	// create command
-	exc := exec.Command(file.Path(), args...) //nolint:gosec // everything is okay
+	exc := exec.Command(binPath, args...) //nolint:gosec // everything is okay
 
 	if !runningInConsole && opts.AllowHidingWindow {
 		// Windows only:

cmds/portmaster-start/service_windows.go

@@ -24,6 +24,7 @@ var (
 		RunE: runAndLogControlError(func(cmd *cobra.Command, args []string) error {
 			return runService(cmd, &Options{
 				Identifier:        "core/portmaster-core",
+				ShortIdentifier:   "core",
 				AllowDownload:     true,
 				AllowHidingWindow: false,
 				NoOutput:          true,

cmds/portmaster-start/show.go

@@ -15,8 +15,8 @@ func init() {
 var showCmd = &cobra.Command{
 	Use: "show",
 	PersistentPreRunE: func(*cobra.Command, []string) error {
-		// all show sub-commands need the data-root but no logging.
+		// All show sub-commands need the registry but no logging.
-		return configureDataRoot(false)
+		return configureRegistry(false)
 	},
 	Short: "Show the command to run a Portmaster component yourself",
 }
@@ -27,7 +27,7 @@ func show(opts *Options, cmdArgs []string) error {
 
 	// adapt identifier
 	if onWindows {
-		opts.Identifier += ".exe"
+		opts.Identifier += exeSuffix
 	}
 
 	file, err := registry.GetFile(platform(opts.Identifier))

cmds/portmaster-start/update.go

@@ -3,22 +3,49 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"runtime"
 
 	"github.com/safing/portbase/log"
 	"github.com/spf13/cobra"
 )
 
+var reset bool
+
 func init() {
-	rootCmd.AddCommand(updatesCmd)
+	rootCmd.AddCommand(updateCmd)
+	rootCmd.AddCommand(purgeCmd)
+
+	flags := updateCmd.Flags()
+	flags.BoolVar(&reset, "reset", false, "Delete all resources and re-download the basic set")
 }
 
-var updatesCmd = &cobra.Command{
+var (
-	Use:   "update",
+	updateCmd = &cobra.Command{
-	Short: "Run a manual update process",
+		Use:   "update",
-	RunE: func(cmd *cobra.Command, args []string) error {
+		Short: "Run a manual update process",
-		return downloadUpdates()
+		RunE: func(cmd *cobra.Command, args []string) error {
-	},
+			return downloadUpdates()
+		},
+	}
+
+	purgeCmd = &cobra.Command{
+		Use:   "purge",
+		Short: "Remove old resource versions that are superseded by at least three versions",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return purge()
+		},
+	}
+)
+
+func indexRequired(cmd *cobra.Command) bool {
+	switch cmd {
+	case updateCmd,
+		purgeCmd:
+		return true
+	default:
+		return false
+	}
 }
 
 func downloadUpdates() error {
@@ -26,8 +53,9 @@ func downloadUpdates() error {
 	if onWindows {
 		registry.MandatoryUpdates = []string{
 			platform("core/portmaster-core.exe"),
+			platform("kext/portmaster-kext.dll"),
+			platform("kext/portmaster-kext.sys"),
 			platform("start/portmaster-start.exe"),
-			platform("app/portmaster-app.exe"),
 			platform("notifier/portmaster-notifier.exe"),
 			platform("notifier/portmaster-snoretoast.exe"),
 		}
@@ -35,7 +63,6 @@ func downloadUpdates() error {
 		registry.MandatoryUpdates = []string{
 			platform("core/portmaster-core"),
 			platform("start/portmaster-start"),
-			platform("app/portmaster-app"),
 			platform("notifier/portmaster-notifier"),
 		}
 	}
@@ -43,10 +70,64 @@ func downloadUpdates() error {
 	// add updates that we require on all platforms.
 	registry.MandatoryUpdates = append(
 		registry.MandatoryUpdates,
-		"all/ui/modules/base.zip",
+		platform("app/portmaster-app.zip"),
+		"all/ui/modules/portmaster.zip",
 	)
 
-	log.SetLogLevel(log.InfoLevel)
+	// Add assets that need unpacking.
+	registry.AutoUnpack = []string{
+		platform("app/portmaster-app.zip"),
+	}
+
+	// logging is configured as a persistent pre-run method inherited from
+	// the root command but since we don't use run.Run() we need to start
+	// logging ourself.
+	log.SetLogLevel(log.TraceLevel)
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+	}
+	defer log.Shutdown()
+
+	if reset {
+		// Delete storage.
+		err = os.RemoveAll(registry.StorageDir().Path)
+		if err != nil {
+			return fmt.Errorf("failed to reset update dir: %s", err)
+		}
+		err = registry.StorageDir().Ensure()
+		if err != nil {
+			return fmt.Errorf("failed to create update dir: %s", err)
+		}
+
+		// Reset registry state.
+		registry.Reset()
+	}
+
+	// Update all indexes.
+	err = registry.UpdateIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// Download all required updates.
+	err = registry.DownloadUpdates(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// Select versions and unpack the selected.
+	registry.SelectVersions()
+	err = registry.UnpackResources()
+	if err != nil {
+		return fmt.Errorf("failed to unpack resources: %s", err)
+	}
+
+	return nil
+}
+
+func purge() error {
+	log.SetLogLevel(log.TraceLevel)
 
 	// logging is configured as a persistent pre-run method inherited from
 	// the root command but since we don't use run.Run() we need to start
@@ -57,7 +138,8 @@ func downloadUpdates() error {
 	}
 	defer log.Shutdown()
 
-	return registry.DownloadUpdates(context.TODO())
+	registry.Purge(3)
+	return nil
 }
 
 func platform(identifier string) string {

cmds/portmaster-start/version.go

@@ -20,9 +20,9 @@ var versionCmd = &cobra.Command{
 	Args:  cobra.NoArgs,
 	PersistentPreRunE: func(*cobra.Command, []string) error {
 		if showAllVersions {
-			// if we are going to show all component versions
+			// If we are going to show all component versions,
-			// we need the dataroot to be configured.
+			// we need the registry to be configured.
-			if err := configureDataRoot(false); err != nil {
+			if err := configureRegistry(false); err != nil {
 				return err
 			}
 		}

cmds/updatemgr/.gitignore

@@ -0,0 +1,2 @@
+updatemgr
+updatemgr.exe

cmds/updatemgr/confirm.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+func confirm(msg string) bool {
+	fmt.Printf("%s: [y|n] ", msg)
+
+	scanner := bufio.NewScanner(os.Stdin)
+	ok := scanner.Scan()
+	if ok && strings.TrimSpace(scanner.Text()) == "y" {
+		return true
+	}
+
+	return false
+}

cmds/updatemgr/main.go

@@ -12,7 +12,7 @@ import (
 var registry *updater.ResourceRegistry
 
 var rootCmd = &cobra.Command{
-	Use:   "uptool",
+	Use:   "updatemgr",
 	Short: "A simple tool to assist in the update and release process",
 	Args:  cobra.ExactArgs(1),
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {

cmds/updatemgr/purge.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/updater"
+)
+
+func init() {
+	rootCmd.AddCommand(purgeCmd)
+}
+
+var purgeCmd = &cobra.Command{
+	Use:   "purge",
+	Short: "Remove old resource versions that are superseded by at least three versions",
+	Args:  cobra.ExactArgs(1),
+	RunE:  purge,
+}
+
+func purge(cmd *cobra.Command, args []string) error {
+	log.SetLogLevel(log.TraceLevel)
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+	}
+	defer log.Shutdown()
+
+	registry.AddIndex(updater.Index{
+		Path:   "stable.json",
+		Stable: true,
+		Beta:   false,
+	})
+
+	registry.AddIndex(updater.Index{
+		Path:   "beta.json",
+		Stable: false,
+		Beta:   true,
+	})
+
+	err = registry.LoadIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	err = scanStorage()
+	if err != nil {
+		return err
+	}
+
+	registry.SelectVersions()
+	registry.Purge(3)
+
+	return nil
+}

cmds/updatemgr/staging.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/safing/portbase/updater"
+	"github.com/spf13/cobra"
+)
+
+var (
+	stageReset bool
+)
+
+func init() {
+	rootCmd.AddCommand(stageCmd)
+	stageCmd.Flags().BoolVar(&stageReset, "reset", false, "Reset staging assets")
+}
+
+var stageCmd = &cobra.Command{
+	Use:   "stage",
+	Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes",
+	Args:  cobra.ExactArgs(1),
+	RunE:  stage,
+}
+
+func stage(cmd *cobra.Command, args []string) error {
+	registry.AddIndex(updater.Index{
+		Path:   "stable.json",
+		Stable: true,
+		Beta:   false,
+	})
+
+	registry.AddIndex(updater.Index{
+		Path:   "beta.json",
+		Stable: false,
+		Beta:   true,
+	})
+
+	err := registry.LoadIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	err = scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// Check if we want to reset staging instead.
+	if stageReset {
+		for _, stagedPath := range exportStaging(true) {
+			err = os.Remove(stagedPath)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	// Export all staged versions and format them.
+	stagingData, err := json.MarshalIndent(exportStaging(false), "", " ")
+	if err != nil {
+		return err
+	}
+
+	// Build destination path.
+	stagingIndexFilePath := filepath.Join(registry.StorageDir().Path, "staging.json")
+
+	// Print preview.
+	fmt.Printf("staging (%s):\n", stagingIndexFilePath)
+	fmt.Println(string(stagingData))
+
+	// Ask for confirmation.
+	if !confirm("\nDo you want to write this index?") {
+		fmt.Println("aborted...")
+		return nil
+	}
+
+	// Write new index to disk.
+	err = ioutil.WriteFile(stagingIndexFilePath, stagingData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", stagingIndexFilePath)
+
+	return nil
+}
+
+func exportStaging(storagePath bool) map[string]string {
+	// Sort all versions.
+	registry.SetBeta(false)
+	registry.SelectVersions()
+	export := registry.Export()
+
+	// Go through all versions and save the highest version, if not stable or beta.
+	versions := make(map[string]string)
+	for _, rv := range export {
+		// Get highest version.
+		v := rv.Versions[0]
+
+		// Do not take stable or beta releases into account.
+		if v.StableRelease || v.BetaRelease {
+			continue
+		}
+
+		// Add highest version to staging
+		if storagePath {
+			rv.SelectedVersion = v
+			versions[rv.Identifier] = rv.GetFile().Path()
+		} else {
+			versions[rv.Identifier] = v.VersionNumber
+		}
+	}
+
+	return versions
+}

cmds/updatemgr/update.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(updateCmd)
+}
+
+var updateCmd = &cobra.Command{
+	Use:   "update",
+	Short: "Update scans the specified directory and registry the index and symlink structure",
+	Args:  cobra.ExactArgs(1),
+	RunE:  update,
+}
+
+func update(cmd *cobra.Command, args []string) error {
+	err := scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// Export versions.
+	betaData, err := json.MarshalIndent(exportSelected(true), "", " ")
+	if err != nil {
+		return err
+	}
+	stableData, err := json.MarshalIndent(exportSelected(false), "", " ")
+	if err != nil {
+		return err
+	}
+
+	// Build destination paths.
+	betaIndexFilePath := filepath.Join(registry.StorageDir().Path, "beta.json")
+	stableIndexFilePath := filepath.Join(registry.StorageDir().Path, "stable.json")
+
+	// Print previews.
+	fmt.Printf("beta (%s):\n", betaIndexFilePath)
+	fmt.Println(string(betaData))
+	fmt.Printf("\nstable: (%s)\n", stableIndexFilePath)
+	fmt.Println(string(stableData))
+
+	// Ask for confirmation.
+	if !confirm("\nDo you want to write these new indexes (and update latest symlinks)?") {
+		fmt.Println("aborted...")
+		return nil
+	}
+
+	// Write indexes.
+	err = ioutil.WriteFile(betaIndexFilePath, betaData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", betaIndexFilePath)
+
+	err = ioutil.WriteFile(stableIndexFilePath, stableData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", stableIndexFilePath)
+
+	// Create symlinks to latest stable versions.
+	symlinksDir := registry.StorageDir().ChildDir("latest", 0o755)
+	err = registry.CreateSymlinks(symlinksDir)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("updated stable symlinks in %s\n", symlinksDir.Path)
+
+	return nil
+}

cmds/uptool/.gitignore

@@ -1 +0,0 @@
-uptool

cmds/uptool/update.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	rootCmd.AddCommand(updateCmd)
-}
-
-var updateCmd = &cobra.Command{
-	Use:   "update",
-	Short: "Update scans the specified directory and registry the index and symlink structure",
-	Args:  cobra.ExactArgs(1),
-	RunE:  update,
-}
-
-func update(cmd *cobra.Command, args []string) error {
-	err := scanStorage()
-	if err != nil {
-		return err
-	}
-
-	// export beta
-	data, err := json.MarshalIndent(exportSelected(true), "", " ")
-	if err != nil {
-		return err
-	}
-	// print
-	fmt.Println("beta:")
-	fmt.Println(string(data))
-	// write index
-	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "beta.json"), data, 0o644) //nolint:gosec // 0644 is intended
-	if err != nil {
-		return err
-	}
-
-	// export stable
-	data, err = json.MarshalIndent(exportSelected(false), "", " ")
-	if err != nil {
-		return err
-	}
-	// print
-	fmt.Println("\nstable:")
-	fmt.Println(string(data))
-	// write index
-	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "stable.json"), data, 0o644) //nolint:gosec // 0644 is intended
-	if err != nil {
-		return err
-	}
-	// create symlinks
-	err = registry.CreateSymlinks(registry.StorageDir().ChildDir("latest", 0o755))
-	if err != nil {
-		return err
-	}
-	fmt.Println("\nstable symlinks created")
-
-	return nil
-}

pack

@@ -3,33 +3,60 @@
 baseDir="$( cd "$(dirname "$0")" && pwd )"
 cd "$baseDir"
 
-# first check what will be built
+COL_OFF="\033[0m"
+COL_BOLD="\033[01;01m"
+COL_RED="\033[31m"
+COL_GREEN="\033[32m"
+COL_YELLOW="\033[33m"
 
-function packAll() {
+function safe_execute {
-  for i in ./cmds/* ; do
+  echo -e "\n[....] $*"
-    if [ -e $i/pack ]; then
+  $*
-      $i/pack $1
+  if [[ $? -eq 0 ]]; then
-    fi
+    echo -e "[${COL_GREEN} OK ${COL_OFF}] $*"
-  done
+  else
+    echo -e "[${COL_RED}FAIL${COL_OFF}] $*" >/dev/stderr
+    echo -e "[${COL_RED}CRIT${COL_OFF}] ABORTING..." >/dev/stderr
+    exit 1
+  fi
 }
 
-echo ""
+function check {
-echo "pack list:"
+  ./cmds/portmaster-core/pack check
-echo ""
+  ./cmds/portmaster-start/pack check
+}
 
-packAll check
+function build {
+  safe_execute ./cmds/portmaster-core/pack build
+  safe_execute ./cmds/portmaster-start/pack build
+}
 
-# confirm
+function reset {
+  ./cmds/portmaster-core/pack reset
+  ./cmds/portmaster-start/pack reset
+}
 
-echo ""
+case $1 in
-read -p "press [Enter] to start packing" x
+  "check" )
-echo ""
+    check
-
+    ;;
-# build
+  "build" )
-
+    build
-set -e
+    ;;
-packAll build
+  "reset" )
-
+    reset
-echo ""
+    ;;
-echo "finished packing."
+  * )
-echo ""
+    echo ""
+    echo "build list:"
+    echo ""
+    check
+    echo ""
+    read -p "press [Enter] to start building" x
+    echo ""
+    build
+    echo ""
+    echo "finished building."
+    echo ""
+    ;;
+esac

ui/serve.go

@@ -28,7 +28,7 @@ func registerRoutes() error {
 	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}", redirAddSlash).Methods("GET", "HEAD")
 	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/", ServeBundle("")).Methods("GET", "HEAD")
 	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", ServeBundle("")).Methods("GET", "HEAD")
-	api.RegisterHandleFunc("/", RedirectToBase)
+	api.RegisterHandleFunc("/", redirectToDefault)
 
 	return nil
 }
@@ -97,13 +97,21 @@ func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName stri
 	readCloser, err := bundle.Open(path)
 	if err != nil {
 		if err == resources.ErrNotFound {
-			log.Tracef("ui: requested resource \"%s\" not found in bundle %s: %s", path, bundleName, err)
+			// Check if there is a base index.html file we can serve instead.
-			http.Error(w, err.Error(), http.StatusNotFound)
+			var indexErr error
+			path = "index.html"
+			readCloser, indexErr = bundle.Open(path)
+			if indexErr != nil {
+				// If we cannot get an index, continue with handling the original error.
+				log.Tracef("ui: requested resource \"%s\" not found in bundle %s: %s", path, bundleName, err)
+				http.Error(w, err.Error(), http.StatusNotFound)
+				return
+			}
 		} else {
 			log.Tracef("ui: error opening module %s: %s", bundleName, err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
-		return
 	}
 
 	// set content type
@@ -131,9 +139,9 @@ func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName stri
 	readCloser.Close()
 }
 
-// RedirectToBase redirects the requests to the control app
+// redirectToDefault redirects the request to the default UI module.
-func RedirectToBase(w http.ResponseWriter, r *http.Request) {
+func redirectToDefault(w http.ResponseWriter, r *http.Request) {
-	u, err := url.Parse("/ui/modules/base/")
+	u, err := url.Parse("/ui/modules/portmaster/")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -141,6 +149,7 @@ func RedirectToBase(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, r.URL.ResolveReference(u).String(), http.StatusTemporaryRedirect)
 }
 
+// redirAddSlash redirects the request to the same, but with a trailing slash.
 func redirAddSlash(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, r.RequestURI+"/", http.StatusPermanentRedirect)
 }

updates/export.go

@@ -35,6 +35,8 @@ type versions struct {
 
 	Core      *info.Info
 	Resources map[string]*updater.Resource
+	Beta      bool
+	Staging   bool
 
 	internalSave bool
 }
@@ -43,6 +45,8 @@ func initVersionExport() (err error) {
 	// init export struct
 	versionExport = &versions{
 		internalSave: true,
+		Beta:         registry.Beta,
+		Staging:      staging,
 	}
 	versionExport.SetKey(versionsDBKey)
 

updates/main.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"os"
+	"path/filepath"
 	"runtime"
 	"time"
 
@@ -49,6 +51,7 @@ var (
 	module            *modules.Module
 	registry          *updater.ResourceRegistry
 	userAgentFromFlag string
+	staging           bool
 
 	updateTask          *modules.Task
 	updateASAP          bool
@@ -76,13 +79,13 @@ func init() {
 	module.RegisterEvent(ResourceUpdateEvent)
 
 	flag.StringVar(&userAgentFromFlag, "update-agent", "", "Sets the user agent for requests to the update server")
+	flag.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)")
 
 	// initialize mandatory updates
 	if onWindows {
 		MandatoryUpdates = []string{
 			platform("core/portmaster-core.exe"),
 			platform("start/portmaster-start.exe"),
-			platform("app/portmaster-app.exe"),
 			platform("notifier/portmaster-notifier.exe"),
 			platform("notifier/portmaster-snoretoast.exe"),
 		}
@@ -90,10 +93,15 @@ func init() {
 		MandatoryUpdates = []string{
 			platform("core/portmaster-core"),
 			platform("start/portmaster-start"),
-			platform("app/portmaster-app"),
 			platform("notifier/portmaster-notifier"),
 		}
 	}
+
+	MandatoryUpdates = append(
+		MandatoryUpdates,
+		platform("app/portmaster-app.zip"),
+		"all/ui/modules/portmaster.zip",
+	)
 }
 
 func prep() error {
@@ -139,9 +147,12 @@ func start() error {
 		},
 		UserAgent:        UserAgent,
 		MandatoryUpdates: MandatoryUpdates,
-		Beta:             releaseChannel() == releaseChannelBeta,
+		AutoUnpack: []string{
-		DevMode:          devMode(),
+			platform("app/portmaster-app.zip"),
-		Online:           true,
+		},
+		Beta:    releaseChannel() == releaseChannelBeta,
+		DevMode: devMode(),
+		Online:  true,
 	}
 	if userAgentFromFlag != "" {
 		// override with flag value
@@ -159,18 +170,33 @@ func start() error {
 		Beta:   false,
 	})
 
-	registry.AddIndex(updater.Index{
+	if registry.Beta {
-		Path:   "beta.json",
+		registry.AddIndex(updater.Index{
-		Stable: false,
+			Path:   "beta.json",
-		Beta:   true,
+			Stable: false,
-	})
+			Beta:   true,
+		})
+	}
 
 	registry.AddIndex(updater.Index{
 		Path:   "all/intel/intel.json",
 		Stable: true,
-		Beta:   false,
+		Beta:   true,
 	})
 
+	if stagingActive() {
+		// Set flag no matter how staging was activated.
+		staging = true
+
+		log.Warning("updates: staging environment is active")
+
+		registry.AddIndex(updater.Index{
+			Path:   "staging.json",
+			Stable: true,
+			Beta:   true,
+		})
+	}
+
 	err = registry.LoadIndexes(module.Ctx)
 	if err != nil {
 		log.Warningf("updates: failed to load indexes: %s", err)
@@ -184,6 +210,7 @@ func start() error {
 	registry.SelectVersions()
 	module.TriggerEvent(VersionUpdateEvent, nil)
 
+	// Initialize the version export - this requires the registry to be set up.
 	err = initVersionExport()
 	if err != nil {
 		return err
@@ -257,7 +284,7 @@ func checkForUpdates(ctx context.Context) (err error) {
 		if err == nil {
 			module.Resolve(updateInProgress)
 		} else {
-			module.Warning(updateFailed, "Failed to check for updates: "+err.Error())
+			module.Warning(updateFailed, "Failed to update: "+err.Error())
 		}
 	}()
 
@@ -273,6 +300,13 @@ func checkForUpdates(ctx context.Context) (err error) {
 
 	registry.SelectVersions()
 
+	// Unpack selected resources.
+	err = registry.UnpackResources()
+	if err != nil {
+		err = fmt.Errorf("failed to update: %w", err)
+		return
+	}
+
 	module.TriggerEvent(ResourceUpdateEvent, nil)
 	return nil
 }
@@ -288,3 +322,14 @@ func stop() error {
 func platform(identifier string) string {
 	return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier)
 }
+
+func stagingActive() bool {
+	// Check flag and env variable.
+	if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" {
+		return true
+	}
+
+	// Check if staging index is present and acessible.
+	_, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json"))
+	return err == nil
+}

updates/upgrader.go

@@ -15,6 +15,7 @@ import (
 	processInfo "github.com/shirou/gopsutil/process"
 	"github.com/tevino/abool"
 
+	"github.com/safing/portbase/dataroot"
 	"github.com/safing/portbase/info"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/notifications"
@@ -206,12 +207,11 @@ func upgradePortmasterStart() error {
 	}
 
 	// update portmaster-start in data root
-	rootPmStartPath := filepath.Join(filepath.Dir(registry.StorageDir().Path), filename)
+	rootPmStartPath := filepath.Join(dataroot.Root().Path, filename)
 	err := upgradeFile(rootPmStartPath, pmCtrlUpdate)
 	if err != nil {
 		return err
 	}
-	log.Infof("updates: upgraded %s", rootPmStartPath)
 
 	return nil
 }
@@ -290,6 +290,7 @@ func upgradeFile(fileToUpgrade string, file *updater.File) error {
 			// abort if version matches
 			currentVersion = strings.Trim(strings.TrimSpace(string(out)), "*")
 			if currentVersion == file.Version() {
+				log.Tracef("updates: %s is already v%s", fileToUpgrade, file.Version())
 				// already up to date!
 				return nil
 			}
@@ -352,6 +353,8 @@ func upgradeFile(fileToUpgrade string, file *updater.File) error {
 			}
 		}
 	}
+
+	log.Infof("updates: upgraded %s to v%s", fileToUpgrade, file.Version())
 	return nil
 }