From 6c9d8535d52b1566713d8733ddf73d9531fadd1c Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Nov 2020 16:45:57 +0100 Subject: [PATCH] Add support for staging and purging --- cmds/portmaster-start/main.go | 48 +++++++++--- cmds/portmaster-start/update.go | 106 ++++++++++++++++++++++--- cmds/portmaster-start/version.go | 6 +- cmds/updatemgr/.gitignore | 2 + cmds/updatemgr/confirm.go | 20 +++++ cmds/{uptool => updatemgr}/main.go | 2 +- cmds/updatemgr/purge.go | 58 ++++++++++++++ cmds/{uptool => updatemgr}/scan.go | 0 cmds/updatemgr/staging.go | 122 +++++++++++++++++++++++++++++ cmds/updatemgr/update.go | 77 ++++++++++++++++++ cmds/uptool/.gitignore | 1 - cmds/uptool/update.go | 64 --------------- updates/export.go | 4 + updates/main.go | 25 ++++++ 14 files changed, 442 insertions(+), 93 deletions(-) create mode 100644 cmds/updatemgr/.gitignore create mode 100644 cmds/updatemgr/confirm.go rename cmds/{uptool => updatemgr}/main.go (96%) create mode 100644 cmds/updatemgr/purge.go rename cmds/{uptool => updatemgr}/scan.go (100%) create mode 100644 cmds/updatemgr/staging.go create mode 100644 cmds/updatemgr/update.go delete mode 100644 cmds/uptool/.gitignore delete mode 100644 cmds/uptool/update.go diff --git a/cmds/portmaster-start/main.go b/cmds/portmaster-start/main.go index 6d736b1c..c1189b3b 100644 --- a/cmds/portmaster-start/main.go +++ b/cmds/portmaster-start/main.go @@ -22,6 +22,7 @@ import ( var ( dataDir string + staging bool maxRetries int dataRoot *utils.DirStructure logsRoot *utils.DirStructure @@ -41,8 +42,8 @@ var ( Use: "portmaster-start", Short: "Start Portmaster components", PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { - mustLoadIndex := cmd == updatesCmd - if err := configureDataRoot(mustLoadIndex); err != nil { + mustLoadIndex := indexRequired(cmd) + if err := configureRegistry(mustLoadIndex); err != nil { return err } @@ -64,8 +65,9 @@ func init() { { flags.StringVar(&dataDir, "data", "", "Configures the data directory. Alternatively, this can also be set via the environment variable PORTMASTER_DATA.") flags.StringVar(®istry.UserAgent, "update-agent", "Start", "Sets the user agent for requests to the update server") + flags.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)") flags.IntVar(&maxRetries, "max-retries", 5, "Maximum number of retries when starting a Portmaster component") - flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdid.") + flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdin.") _ = rootCmd.MarkPersistentFlagDirname("data") _ = flags.MarkHidden("input-signals") } @@ -131,34 +133,32 @@ func initCobra() { portlog.SetLogLevel(portlog.CriticalLevel) } -func configureDataRoot(mustLoadIndex bool) error { - // The data directory is not - // check for environment variable - // PORTMASTER_DATA +func configureRegistry(mustLoadIndex bool) error { + // If dataDir is not set, check the environment variable. if dataDir == "" { dataDir = os.Getenv("PORTMASTER_DATA") } - // if it's still empty try to auto-detect it + // If it's still empty, try to auto-detect it. if dataDir == "" { dataDir = detectInstallationDir() } - // finally, if it's still empty the user must provide it + // Finally, if it's still empty, the user must provide it. if dataDir == "" { return errors.New("please set the data directory using --data=/path/to/data/dir") } - // remove redundant escape characters and quotes + // Remove left over quotes. dataDir = strings.Trim(dataDir, `\"`) - // initialize dataroot + // Initialize data root. err := dataroot.Initialize(dataDir, 0755) if err != nil { return fmt.Errorf("failed to initialize data root: %s", err) } dataRoot = dataroot.Root() - // initialize registry + // Initialize registry. err = registry.Initialize(dataRoot.ChildDir("updates", 0755)) if err != nil { return err @@ -177,6 +177,19 @@ func configureDataRoot(mustLoadIndex bool) error { // Beta: true, // }) + if stagingActive() { + // Set flag no matter how staging was activated. + staging = true + + log.Println("WARNING: staging environment is active.") + + registry.AddIndex(updater.Index{ + Path: "staging.json", + Stable: true, + Beta: true, + }) + } + return updateRegistryIndex(mustLoadIndex) } @@ -233,3 +246,14 @@ func detectInstallationDir() string { return parent } + +func stagingActive() bool { + // Check flag and env variable. + if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" { + return true + } + + // Check if staging index is present and acessible. + _, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json")) + return err == nil +} diff --git a/cmds/portmaster-start/update.go b/cmds/portmaster-start/update.go index 84fbb3d7..562c7877 100644 --- a/cmds/portmaster-start/update.go +++ b/cmds/portmaster-start/update.go @@ -3,22 +3,49 @@ package main import ( "context" "fmt" + "os" "runtime" "github.com/safing/portbase/log" "github.com/spf13/cobra" ) +var reset bool + func init() { - rootCmd.AddCommand(updatesCmd) + rootCmd.AddCommand(updateCmd) + rootCmd.AddCommand(purgeCmd) + + flags := updateCmd.Flags() + flags.BoolVar(&reset, "reset", false, "Delete all resources and re-download the basic set") } -var updatesCmd = &cobra.Command{ - Use: "update", - Short: "Run a manual update process", - RunE: func(cmd *cobra.Command, args []string) error { - return downloadUpdates() - }, +var ( + updateCmd = &cobra.Command{ + Use: "update", + Short: "Run a manual update process", + RunE: func(cmd *cobra.Command, args []string) error { + return downloadUpdates() + }, + } + + purgeCmd = &cobra.Command{ + Use: "purge", + Short: "Remove old resource versions that are superseded by at least three versions", + RunE: func(cmd *cobra.Command, args []string) error { + return purge() + }, + } +) + +func indexRequired(cmd *cobra.Command) bool { + switch cmd { + case updateCmd, + purgeCmd: + return true + default: + return false + } } func downloadUpdates() error { @@ -26,8 +53,9 @@ func downloadUpdates() error { if onWindows { registry.MandatoryUpdates = []string{ platform("core/portmaster-core.exe"), + platform("kext/portmaster-kext.dll"), + platform("kext/portmaster-kext.sys"), platform("start/portmaster-start.exe"), - platform("app/portmaster-app.exe"), platform("notifier/portmaster-notifier.exe"), platform("notifier/portmaster-snoretoast.exe"), } @@ -35,7 +63,6 @@ func downloadUpdates() error { registry.MandatoryUpdates = []string{ platform("core/portmaster-core"), platform("start/portmaster-start"), - platform("app/portmaster-app"), platform("notifier/portmaster-notifier"), } } @@ -43,10 +70,64 @@ func downloadUpdates() error { // add updates that we require on all platforms. registry.MandatoryUpdates = append( registry.MandatoryUpdates, - "all/ui/modules/base.zip", + platform("app/portmaster-app.zip"), + "all/ui/modules/portmaster.zip", ) - log.SetLogLevel(log.InfoLevel) + // Add assets that need unpacking. + registry.AutoUnpack = []string{ + platform("app/portmaster-app.zip"), + } + + // logging is configured as a persistent pre-run method inherited from + // the root command but since we don't use run.Run() we need to start + // logging ourself. + log.SetLogLevel(log.TraceLevel) + err := log.Start() + if err != nil { + fmt.Printf("failed to start logging: %s\n", err) + } + defer log.Shutdown() + + if reset { + // Delete storage. + err = os.RemoveAll(registry.StorageDir().Path) + if err != nil { + return fmt.Errorf("failed to reset update dir: %s", err) + } + err = registry.StorageDir().Ensure() + if err != nil { + return fmt.Errorf("failed to create update dir: %s", err) + } + + // Reset registry state. + registry.Reset() + } + + // Update all indexes. + err = registry.UpdateIndexes(context.TODO()) + if err != nil { + return err + } + + // Download all required updates. + err = registry.DownloadUpdates(context.TODO()) + if err != nil { + return err + } + + // Select versions and unpack the selected. + registry.SelectVersions() + err = registry.UnpackResources() + if err != nil { + return fmt.Errorf("failed to unpack resources: %s", err) + } + + return nil +} + +func purge() error { + log.SetLogLevel(log.TraceLevel) // logging is configured as a persistent pre-run method inherited from // the root command but since we don't use run.Run() we need to start @@ -57,7 +138,8 @@ func downloadUpdates() error { } defer log.Shutdown() - return registry.DownloadUpdates(context.TODO()) + registry.Purge(3) + return nil } func platform(identifier string) string { diff --git a/cmds/portmaster-start/version.go b/cmds/portmaster-start/version.go index 8b19b673..9f3dc662 100644 --- a/cmds/portmaster-start/version.go +++ b/cmds/portmaster-start/version.go @@ -20,9 +20,9 @@ var versionCmd = &cobra.Command{ Args: cobra.NoArgs, PersistentPreRunE: func(*cobra.Command, []string) error { if showAllVersions { - // if we are going to show all component versions - // we need the dataroot to be configured. - if err := configureDataRoot(false); err != nil { + // If we are going to show all component versions, + // we need the registry to be configured. + if err := configureRegistry(false); err != nil { return err } } diff --git a/cmds/updatemgr/.gitignore b/cmds/updatemgr/.gitignore new file mode 100644 index 00000000..3f56c4be --- /dev/null +++ b/cmds/updatemgr/.gitignore @@ -0,0 +1,2 @@ +updatemgr +updatemgr.exe diff --git a/cmds/updatemgr/confirm.go b/cmds/updatemgr/confirm.go new file mode 100644 index 00000000..293faaf6 --- /dev/null +++ b/cmds/updatemgr/confirm.go @@ -0,0 +1,20 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +func confirm(msg string) bool { + fmt.Printf("%s: [y|n] ", msg) + + scanner := bufio.NewScanner(os.Stdin) + ok := scanner.Scan() + if ok && strings.TrimSpace(scanner.Text()) == "y" { + return true + } + + return false +} diff --git a/cmds/uptool/main.go b/cmds/updatemgr/main.go similarity index 96% rename from cmds/uptool/main.go rename to cmds/updatemgr/main.go index fe22d78d..a3de4539 100644 --- a/cmds/uptool/main.go +++ b/cmds/updatemgr/main.go @@ -12,7 +12,7 @@ import ( var registry *updater.ResourceRegistry var rootCmd = &cobra.Command{ - Use: "uptool", + Use: "updatemgr", Short: "A simple tool to assist in the update and release process", Args: cobra.ExactArgs(1), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmds/updatemgr/purge.go b/cmds/updatemgr/purge.go new file mode 100644 index 00000000..7fb715d0 --- /dev/null +++ b/cmds/updatemgr/purge.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/safing/portbase/log" + "github.com/safing/portbase/updater" +) + +func init() { + rootCmd.AddCommand(purgeCmd) +} + +var purgeCmd = &cobra.Command{ + Use: "purge", + Short: "Remove old resource versions that are superseded by at least three versions", + Args: cobra.ExactArgs(1), + RunE: purge, +} + +func purge(cmd *cobra.Command, args []string) error { + log.SetLogLevel(log.TraceLevel) + err := log.Start() + if err != nil { + fmt.Printf("failed to start logging: %s\n", err) + } + defer log.Shutdown() + + registry.AddIndex(updater.Index{ + Path: "stable.json", + Stable: true, + Beta: false, + }) + + registry.AddIndex(updater.Index{ + Path: "beta.json", + Stable: false, + Beta: true, + }) + + err = registry.LoadIndexes(context.TODO()) + if err != nil { + return err + } + + err = scanStorage() + if err != nil { + return err + } + + registry.SelectVersions() + registry.Purge(3) + + return nil +} diff --git a/cmds/uptool/scan.go b/cmds/updatemgr/scan.go similarity index 100% rename from cmds/uptool/scan.go rename to cmds/updatemgr/scan.go diff --git a/cmds/updatemgr/staging.go b/cmds/updatemgr/staging.go new file mode 100644 index 00000000..0fdbb486 --- /dev/null +++ b/cmds/updatemgr/staging.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/safing/portbase/updater" + "github.com/spf13/cobra" +) + +var ( + stageReset bool +) + +func init() { + rootCmd.AddCommand(stageCmd) + stageCmd.Flags().BoolVar(&stageReset, "reset", false, "Reset staging assets") +} + +var stageCmd = &cobra.Command{ + Use: "stage", + Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes", + Args: cobra.ExactArgs(1), + RunE: stage, +} + +func stage(cmd *cobra.Command, args []string) error { + registry.AddIndex(updater.Index{ + Path: "stable.json", + Stable: true, + Beta: false, + }) + + registry.AddIndex(updater.Index{ + Path: "beta.json", + Stable: false, + Beta: true, + }) + + err := registry.LoadIndexes(context.TODO()) + if err != nil { + return err + } + + err = scanStorage() + if err != nil { + return err + } + + // Check if we want to reset staging instead. + if stageReset { + for _, stagedPath := range exportStaging(true) { + err = os.Remove(stagedPath) + if err != nil { + return err + } + } + + return nil + } + + // Export all staged versions and format them. + stagingData, err := json.MarshalIndent(exportStaging(false), "", " ") + if err != nil { + return err + } + + // Build destination path. + stagingIndexFilePath := filepath.Join(registry.StorageDir().Path, "staging.json") + + // Print preview. + fmt.Printf("staging (%s):\n", stagingIndexFilePath) + fmt.Println(string(stagingData)) + + // Ask for confirmation. + if !confirm("\nDo you want to write this index?") { + fmt.Println("aborted...") + return nil + } + + // Write new index to disk. + err = ioutil.WriteFile(stagingIndexFilePath, stagingData, 0o644) //nolint:gosec // 0644 is intended + if err != nil { + return err + } + fmt.Printf("written %s\n", stagingIndexFilePath) + + return nil +} + +func exportStaging(storagePath bool) map[string]string { + // Sort all versions. + registry.SetBeta(false) + registry.SelectVersions() + export := registry.Export() + + // Go through all versions and save the highest version, if not stable or beta. + versions := make(map[string]string) + for _, rv := range export { + // Get highest version. + v := rv.Versions[0] + + // Do not take stable or beta releases into account. + if v.StableRelease || v.BetaRelease { + continue + } + + // Add highest version to staging + if storagePath { + rv.SelectedVersion = v + versions[rv.Identifier] = rv.GetFile().Path() + } else { + versions[rv.Identifier] = v.VersionNumber + } + } + + return versions +} diff --git a/cmds/updatemgr/update.go b/cmds/updatemgr/update.go new file mode 100644 index 00000000..4cf35a26 --- /dev/null +++ b/cmds/updatemgr/update.go @@ -0,0 +1,77 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(updateCmd) +} + +var updateCmd = &cobra.Command{ + Use: "update", + Short: "Update scans the specified directory and registry the index and symlink structure", + Args: cobra.ExactArgs(1), + RunE: update, +} + +func update(cmd *cobra.Command, args []string) error { + err := scanStorage() + if err != nil { + return err + } + + // Export versions. + betaData, err := json.MarshalIndent(exportSelected(true), "", " ") + if err != nil { + return err + } + stableData, err := json.MarshalIndent(exportSelected(false), "", " ") + if err != nil { + return err + } + + // Build destination paths. + betaIndexFilePath := filepath.Join(registry.StorageDir().Path, "beta.json") + stableIndexFilePath := filepath.Join(registry.StorageDir().Path, "stable.json") + + // Print previews. + fmt.Printf("beta (%s):\n", betaIndexFilePath) + fmt.Println(string(betaData)) + fmt.Printf("\nstable: (%s)\n", stableIndexFilePath) + fmt.Println(string(stableData)) + + // Ask for confirmation. + if !confirm("\nDo you want to write these new indexes (and update latest symlinks)?") { + fmt.Println("aborted...") + return nil + } + + // Write indexes. + err = ioutil.WriteFile(betaIndexFilePath, betaData, 0o644) //nolint:gosec // 0644 is intended + if err != nil { + return err + } + fmt.Printf("written %s\n", betaIndexFilePath) + + err = ioutil.WriteFile(stableIndexFilePath, stableData, 0o644) //nolint:gosec // 0644 is intended + if err != nil { + return err + } + fmt.Printf("written %s\n", stableIndexFilePath) + + // Create symlinks to latest stable versions. + symlinksDir := registry.StorageDir().ChildDir("latest", 0o755) + err = registry.CreateSymlinks(symlinksDir) + if err != nil { + return err + } + fmt.Printf("updated stable symlinks in %s\n", symlinksDir.Path) + + return nil +} diff --git a/cmds/uptool/.gitignore b/cmds/uptool/.gitignore deleted file mode 100644 index c5074cf6..00000000 --- a/cmds/uptool/.gitignore +++ /dev/null @@ -1 +0,0 @@ -uptool diff --git a/cmds/uptool/update.go b/cmds/uptool/update.go deleted file mode 100644 index 442c31eb..00000000 --- a/cmds/uptool/update.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "path/filepath" - - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(updateCmd) -} - -var updateCmd = &cobra.Command{ - Use: "update", - Short: "Update scans the specified directory and registry the index and symlink structure", - Args: cobra.ExactArgs(1), - RunE: update, -} - -func update(cmd *cobra.Command, args []string) error { - err := scanStorage() - if err != nil { - return err - } - - // export beta - data, err := json.MarshalIndent(exportSelected(true), "", " ") - if err != nil { - return err - } - // print - fmt.Println("beta:") - fmt.Println(string(data)) - // write index - err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "beta.json"), data, 0o644) //nolint:gosec // 0644 is intended - if err != nil { - return err - } - - // export stable - data, err = json.MarshalIndent(exportSelected(false), "", " ") - if err != nil { - return err - } - // print - fmt.Println("\nstable:") - fmt.Println(string(data)) - // write index - err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "stable.json"), data, 0o644) //nolint:gosec // 0644 is intended - if err != nil { - return err - } - // create symlinks - err = registry.CreateSymlinks(registry.StorageDir().ChildDir("latest", 0o755)) - if err != nil { - return err - } - fmt.Println("\nstable symlinks created") - - return nil -} diff --git a/updates/export.go b/updates/export.go index 37872e55..112413b3 100644 --- a/updates/export.go +++ b/updates/export.go @@ -35,6 +35,8 @@ type versions struct { Core *info.Info Resources map[string]*updater.Resource + Beta bool + Staging bool internalSave bool } @@ -43,6 +45,8 @@ func initVersionExport() (err error) { // init export struct versionExport = &versions{ internalSave: true, + Beta: registry.Beta, + Staging: staging, } versionExport.SetKey(versionsDBKey) diff --git a/updates/main.go b/updates/main.go index 53793d0b..db1cf204 100644 --- a/updates/main.go +++ b/updates/main.go @@ -51,6 +51,7 @@ var ( module *modules.Module registry *updater.ResourceRegistry userAgentFromFlag string + staging bool updateTask *modules.Task updateASAP bool @@ -78,6 +79,7 @@ func init() { module.RegisterEvent(ResourceUpdateEvent) flag.StringVar(&userAgentFromFlag, "update-agent", "", "Sets the user agent for requests to the update server") + flag.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)") // initialize mandatory updates if onWindows { @@ -182,6 +184,18 @@ func start() error { Beta: true, }) + if stagingActive() { + // Set flag no matter how staging was activated. + staging = true + + log.Warning("updates: staging environment is active") + + registry.AddIndex(updater.Index{ + Path: "staging.json", + Stable: true, + Beta: true, + }) + } err = registry.LoadIndexes(module.Ctx) if err != nil { @@ -308,3 +322,14 @@ func stop() error { func platform(identifier string) string { return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier) } + +func stagingActive() bool { + // Check flag and env variable. + if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" { + return true + } + + // Check if staging index is present and acessible. + _, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json")) + return err == nil +}