From 8a46e78c0f5f05129907da2a81949922d0a66b4b Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 2 Jun 2021 10:56:02 +0200 Subject: [PATCH 1/7] Fail update check when the updating indexes fails --- updates/main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/updates/main.go b/updates/main.go index 5f2000c6..b3057588 100644 --- a/updates/main.go +++ b/updates/main.go @@ -293,7 +293,7 @@ func checkForUpdates(ctx context.Context) (err error) { notifications.NotifyWarn( updateFailed, "Update Check Failed", - "The Portmaster failed to check for updates. This might be a temporary issue of your device, your network or the update servers. The Portmaster will automatically try again later.", + "The Portmaster failed to check for updates. This might be a temporary issue of your device, your network or the update servers. The Portmaster will automatically try again later. If you just installed the Portmaster, please try disabling potentially conflicting software, such as other firewalls or VPNs.", notifications.Action{ ID: "retry", Text: "Try Again Now", @@ -308,12 +308,13 @@ func checkForUpdates(ctx context.Context) (err error) { }() if err = registry.UpdateIndexes(ctx); err != nil { - log.Warningf("updates: failed to update indexes: %s", err) + err = fmt.Errorf("failed to update indexes: %s", err) + return } err = registry.DownloadUpdates(ctx) if err != nil { - err = fmt.Errorf("failed to update: %w", err) + err = fmt.Errorf("failed to download updates: %w", err) return } @@ -322,7 +323,7 @@ func checkForUpdates(ctx context.Context) (err error) { // Unpack selected resources. err = registry.UnpackResources() if err != nil { - err = fmt.Errorf("failed to update: %w", err) + err = fmt.Errorf("failed to unpack updates: %w", err) return } From 642e2eb11259dffa1f6620f3c962992ef6fb4a65 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 3 Jun 2021 23:28:41 +0200 Subject: [PATCH 2/7] Establish beta and staging release channels, move indexes to helper --- updates/config.go | 28 ++++++---- updates/export.go | 12 +++-- updates/helper/electron.go | 9 ---- updates/helper/indexes.go | 50 ++++++++++++++++++ updates/helper/updates.go | 68 ++++++++++++++++++++++++ updates/main.go | 105 +++++-------------------------------- updates/upgrader.go | 9 ++-- 7 files changed, 161 insertions(+), 120 deletions(-) create mode 100644 updates/helper/indexes.go create mode 100644 updates/helper/updates.go diff --git a/updates/config.go b/updates/config.go index 2889029c..8043038b 100644 --- a/updates/config.go +++ b/updates/config.go @@ -4,6 +4,7 @@ import ( "context" "github.com/safing/portbase/notifications" + "github.com/safing/portmaster/updates/helper" "github.com/tevino/abool" "github.com/safing/portbase/config" @@ -20,6 +21,7 @@ var ( devMode config.BoolOption enableUpdates config.BoolOption + initialReleaseChannel string previousReleaseChannel string updatesCurrentlyEnabled bool previousDevMode bool @@ -29,21 +31,28 @@ var ( func registerConfig() error { err := config.Register(&config.Option{ Name: "Release Channel", - Key: releaseChannelKey, + Key: helper.ReleaseChannelKey, Description: "Switch release channel.", OptType: config.OptTypeString, ExpertiseLevel: config.ExpertiseLevelDeveloper, ReleaseLevel: config.ReleaseLevelExperimental, - RequiresRestart: false, - DefaultValue: releaseChannelStable, + RequiresRestart: true, + DefaultValue: helper.ReleaseChannelStable, PossibleValues: []config.PossibleValue{ { - Name: "Stable", - Value: releaseChannelStable, + Name: "Stable", + Description: "Production releases.", + Value: helper.ReleaseChannelStable, }, { - Name: "Beta", - Value: releaseChannelBeta, + Name: "Beta", + Description: "Production releases for testing new features that may break and cause interruption.", + Value: helper.ReleaseChannelBeta, + }, + { + Name: "Staging", + Description: "Development releases for testing random things and experimenting. Dangerous - only use when told so.", + Value: helper.ReleaseChannelStaging, }, }, Annotations: config.Annotations{ @@ -78,7 +87,8 @@ func registerConfig() error { } func initConfig() { - releaseChannel = config.GetAsString(releaseChannelKey, releaseChannelStable) + releaseChannel = config.GetAsString(helper.ReleaseChannelKey, helper.ReleaseChannelStable) + initialReleaseChannel = releaseChannel() previousReleaseChannel = releaseChannel() enableUpdates = config.GetAsBool(enableUpdatesKey, true) @@ -107,7 +117,7 @@ func updateRegistryConfig(_ context.Context, _ interface{}) error { changed := false if releaseChannel() != previousReleaseChannel { - registry.SetBeta(releaseChannel() == releaseChannelBeta) + registry.SetUsePreReleases(releaseChannel() != helper.ReleaseChannelStable) previousReleaseChannel = releaseChannel() changed = true } diff --git a/updates/export.go b/updates/export.go index 112413b3..e00dee33 100644 --- a/updates/export.go +++ b/updates/export.go @@ -11,14 +11,14 @@ import ( "github.com/safing/portbase/info" "github.com/safing/portbase/log" "github.com/safing/portbase/updater" + "github.com/safing/portmaster/updates/helper" ) -// database key for update information +// Database key for update information. const ( versionsDBKey = "core:status/versions" ) -// working vars var ( versionExport *versions versionExportDB = database.NewInterface(&database.Options{ @@ -35,6 +35,7 @@ type versions struct { Core *info.Info Resources map[string]*updater.Resource + Channel string Beta bool Staging bool @@ -45,8 +46,9 @@ func initVersionExport() (err error) { // init export struct versionExport = &versions{ internalSave: true, - Beta: registry.Beta, - Staging: staging, + Channel: initialReleaseChannel, + Beta: initialReleaseChannel == helper.ReleaseChannelBeta, + Staging: initialReleaseChannel == helper.ReleaseChannelStaging, } versionExport.SetKey(versionsDBKey) @@ -68,7 +70,7 @@ func stopVersionExport() error { return versionExportHook.Cancel() } -// export is an event hook +// export is an event hook. func export(_ context.Context, _ interface{}) error { // populate versionExport.lock.Lock() diff --git a/updates/helper/electron.go b/updates/helper/electron.go index a01dabba..cb997667 100644 --- a/updates/helper/electron.go +++ b/updates/helper/electron.go @@ -1,7 +1,6 @@ package helper import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -54,14 +53,6 @@ func EnsureChromeSandboxPermissions(reg *updater.ResourceRegistry) error { return nil } -// PlatformIdentifier converts identifier for the current platform. -func PlatformIdentifier(identifier string) string { - // From https://golang.org/pkg/runtime/#GOARCH - // GOOS is the running program's operating system target: one of darwin, freebsd, linux, and so on. - // GOARCH is the running program's architecture target: one of 386, amd64, arm, s390x, and so on. - return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier) -} - func checkSysctl(setting string, value byte) bool { c, err := sysctl(setting) if err != nil { diff --git a/updates/helper/indexes.go b/updates/helper/indexes.go new file mode 100644 index 00000000..97689db4 --- /dev/null +++ b/updates/helper/indexes.go @@ -0,0 +1,50 @@ +package helper + +import ( + "github.com/safing/portbase/updater" +) + +const ( + ReleaseChannelKey = "core/releaseChannel" + ReleaseChannelJSONKey = "core.releaseChannel" + ReleaseChannelStable = "stable" + ReleaseChannelBeta = "beta" + ReleaseChannelStaging = "staging" +) + +func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string) { + // Be reminded that the order is important, as indexes added later will + // override the current release from earlier indexes. + + // Reset indexes before adding them (again). + registry.ResetIndexes() + + // Always add the stable index as a base. + registry.AddIndex(updater.Index{ + Path: "stable.json", + }) + + // Add beta index if in beta or staging channel. + if releaseChannel == ReleaseChannelBeta || + releaseChannel == ReleaseChannelStaging { + registry.AddIndex(updater.Index{ + Path: "beta.json", + PreRelease: true, + }) + } + + // Add staging index if in staging channel. + if releaseChannel == ReleaseChannelStaging { + registry.AddIndex(updater.Index{ + Path: "staging.json", + PreRelease: true, + }) + } + + // Add the intel index last, as it updates the fastest and should not be + // crippled by other faulty indexes. It can only specify versions for its + // scope anyway. + registry.AddIndex(updater.Index{ + Path: "all/intel/intel.json", + }) +} diff --git a/updates/helper/updates.go b/updates/helper/updates.go new file mode 100644 index 00000000..6ad4ffe5 --- /dev/null +++ b/updates/helper/updates.go @@ -0,0 +1,68 @@ +package helper + +import ( + "fmt" + "runtime" +) + +const ( + onWindows = runtime.GOOS == "windows" +) + +// PlatformIdentifier converts identifier for the current platform. +func PlatformIdentifier(identifier string) string { + // From https://golang.org/pkg/runtime/#GOARCH + // GOOS is the running program's operating system target: one of darwin, freebsd, linux, and so on. + // GOARCH is the running program's architecture target: one of 386, amd64, arm, s390x, and so on. + return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier) +} + +// MandatoryUpdates returns mandatory updates that should be loaded on install +// or reset. +func MandatoryUpdates() (identifiers []string) { + // Binaries + if onWindows { + identifiers = []string{ + PlatformIdentifier("core/portmaster-core.exe"), + PlatformIdentifier("kext/portmaster-kext.dll"), + PlatformIdentifier("kext/portmaster-kext.sys"), + PlatformIdentifier("start/portmaster-start.exe"), + PlatformIdentifier("notifier/portmaster-notifier.exe"), + PlatformIdentifier("notifier/portmaster-snoretoast.exe"), + } + } else { + identifiers = []string{ + PlatformIdentifier("core/portmaster-core"), + PlatformIdentifier("start/portmaster-start"), + PlatformIdentifier("notifier/portmaster-notifier"), + } + } + + // Components, Assets and Data + identifiers = append( + identifiers, + + // User interface components + PlatformIdentifier("app/portmaster-app.zip"), + "all/ui/modules/portmaster.zip", + "all/ui/modules/assets.zip", + + // Filter lists data + "all/intel/lists/base.dsdl", + "all/intel/lists/intermediate.dsdl", + "all/intel/lists/urgent.dsdl", + + // Geo IP data + "all/intel/geoip/geoipv4.mmdb.gz", + "all/intel/geoip/geoipv6.mmdb.gz", + ) + + return identifiers +} + +// AutoUnpackUpdates returns assets that need unpacking. +func AutoUnpackUpdates() []string { + return []string{ + PlatformIdentifier("app/portmaster-app.zip"), + } +} diff --git a/updates/main.go b/updates/main.go index b3057588..c1d88814 100644 --- a/updates/main.go +++ b/updates/main.go @@ -4,11 +4,11 @@ import ( "context" "flag" "fmt" - "os" - "path/filepath" "runtime" "time" + "github.com/safing/portmaster/updates/helper" + "github.com/safing/portbase/dataroot" "github.com/safing/portbase/log" "github.com/safing/portbase/modules" @@ -19,10 +19,6 @@ import ( const ( onWindows = runtime.GOOS == "windows" - releaseChannelKey = "core/releaseChannel" - releaseChannelStable = "stable" - releaseChannelBeta = "beta" - enableUpdatesKey = "core/automaticUpdates" // ModuleName is the name of the update module @@ -48,16 +44,11 @@ var ( module *modules.Module registry *updater.ResourceRegistry userAgentFromFlag string - staging bool updateTask *modules.Task updateASAP bool disableTaskSchedule bool - // MandatoryUpdates is a list of full identifiers that - // should always be kept up to date. - MandatoryUpdates []string - // UserAgent is an HTTP User-Agent that is used to add // more context to requests made by the registry when // fetching resources from the update server. @@ -65,9 +56,8 @@ var ( ) const ( - updateInProgress = "updates:in-progress" - updateFailed = "updates:failed" - updateSuccess = "updates:success" + updateFailed = "updates:failed" + updateSuccess = "updates:success" ) func init() { @@ -76,29 +66,9 @@ func init() { module.RegisterEvent(ResourceUpdateEvent, true) flag.StringVar(&userAgentFromFlag, "update-agent", "", "set the user agent for requests to the update server") - flag.BoolVar(&staging, "staging", false, "use staging update channel; for testing only") - // initialize mandatory updates - if onWindows { - MandatoryUpdates = []string{ - platform("core/portmaster-core.exe"), - platform("start/portmaster-start.exe"), - platform("notifier/portmaster-notifier.exe"), - platform("notifier/portmaster-snoretoast.exe"), - } - } else { - MandatoryUpdates = []string{ - platform("core/portmaster-core"), - platform("start/portmaster-start"), - platform("notifier/portmaster-notifier"), - } - } - - MandatoryUpdates = append( - MandatoryUpdates, - platform("app/portmaster-app.zip"), - "all/ui/modules/portmaster.zip", - ) + var dummy bool + flag.BoolVar(&dummy, "staging", false, "deprecated, configure in settings instead") } func prep() error { @@ -107,8 +77,6 @@ func prep() error { } return registerAPIEndpoints() - - return nil } func start() error { @@ -131,13 +99,11 @@ func start() error { "https://updates.safing.io", }, UserAgent: UserAgent, - MandatoryUpdates: MandatoryUpdates, - AutoUnpack: []string{ - platform("app/portmaster-app.zip"), - }, - Beta: releaseChannel() == releaseChannelBeta, - DevMode: devMode(), - Online: true, + MandatoryUpdates: helper.MandatoryUpdates(), + AutoUnpack: helper.AutoUnpackUpdates(), + UsePreReleases: initialReleaseChannel != helper.ReleaseChannelStable, + DevMode: devMode(), + Online: true, } if userAgentFromFlag != "" { // override with flag value @@ -149,38 +115,8 @@ func start() error { return err } - registry.AddIndex(updater.Index{ - Path: "stable.json", - Stable: true, - Beta: false, - }) - - if registry.Beta { - registry.AddIndex(updater.Index{ - Path: "beta.json", - Stable: false, - Beta: true, - }) - } - - registry.AddIndex(updater.Index{ - Path: "all/intel/intel.json", - Stable: true, - Beta: true, - }) - - if stagingActive() { - // Set flag no matter how staging was activated. - staging = true - - log.Warning("updates: staging environment is active") - - registry.AddIndex(updater.Index{ - Path: "staging.json", - Stable: true, - Beta: true, - }) - } + // Set indexes based on the release channel. + helper.SetIndexes(registry, initialReleaseChannel) err = registry.LoadIndexes(module.Ctx) if err != nil { @@ -342,21 +278,6 @@ func stop() error { return stopVersionExport() } -func platform(identifier string) string { - return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier) -} - -func stagingActive() bool { - // Check flag and env variable. - if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" { - return true - } - - // Check if staging index is present and acessible. - _, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json")) - return err == nil -} - // RootPath returns the root path used for storing updates. func RootPath() string { if !module.Online() { diff --git a/updates/upgrader.go b/updates/upgrader.go index 83c5aaa2..753bddde 100644 --- a/updates/upgrader.go +++ b/updates/upgrader.go @@ -351,10 +351,9 @@ func upgradeFile(fileToUpgrade string, file *updater.File) error { } // CopyFile atomically copies a file using the update registry's tmp dir. -func CopyFile(srcPath, dstPath string) (err error) { - +func CopyFile(srcPath, dstPath string) error { // check tmp dir - err = registry.TmpDir().Ensure() + err := registry.TmpDir().Ensure() if err != nil { return fmt.Errorf("could not prepare tmp directory for copying file: %w", err) } @@ -369,14 +368,14 @@ func CopyFile(srcPath, dstPath string) (err error) { // open source srcFile, err := os.Open(srcPath) if err != nil { - return + return err } defer srcFile.Close() // copy data _, err = io.Copy(atomicDstFile, srcFile) if err != nil { - return + return err } // finalize file From 2127f1b2109e6ef5a5c174bd78ac1c1256cc1773 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 3 Jun 2021 23:30:01 +0200 Subject: [PATCH 3/7] Reload indexes when restarting with portmaster-start --- cmds/portmaster-start/main.go | 69 ++++++++++++++++----------------- cmds/portmaster-start/update.go | 39 +++---------------- go.mod | 2 +- 3 files changed, 41 insertions(+), 69 deletions(-) diff --git a/cmds/portmaster-start/main.go b/cmds/portmaster-start/main.go index fd83d94c..35f69482 100644 --- a/cmds/portmaster-start/main.go +++ b/cmds/portmaster-start/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io/ioutil" "log" "os" "os/signal" @@ -11,6 +12,10 @@ import ( "strings" "syscall" + "github.com/safing/portmaster/updates/helper" + + "github.com/tidwall/gjson" + "github.com/safing/portbase/dataroot" "github.com/safing/portbase/info" portlog "github.com/safing/portbase/log" @@ -33,7 +38,6 @@ var ( UpdateURLs: []string{ "https://updates.safing.io", }, - Beta: false, DevMode: false, Online: true, // is disabled later based on command } @@ -65,7 +69,7 @@ func init() { { flags.StringVar(&dataDir, "data", "", "Configures the data directory. Alternatively, this can also be set via the environment variable PORTMASTER_DATA.") flags.StringVar(®istry.UserAgent, "update-agent", "Start", "Sets the user agent for requests to the update server") - flags.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)") + flags.BoolVar(&staging, "staging", false, "Deprecated, configure in settings instead.") flags.IntVar(&maxRetries, "max-retries", 5, "Maximum number of retries when starting a Portmaster component") flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdin.") _ = rootCmd.MarkPersistentFlagDirname("data") @@ -164,32 +168,6 @@ func configureRegistry(mustLoadIndex bool) error { return err } - registry.AddIndex(updater.Index{ - Path: "stable.json", - Stable: true, - Beta: false, - }) - - // TODO: enable loading beta versions - // registry.AddIndex(updater.Index{ - // Path: "beta.json", - // Stable: false, - // Beta: true, - // }) - - if stagingActive() { - // Set flag no matter how staging was activated. - staging = true - - log.Println("WARNING: staging environment is active.") - - registry.AddIndex(updater.Index{ - Path: "staging.json", - Stable: true, - Beta: true, - }) - } - return updateRegistryIndex(mustLoadIndex) } @@ -210,6 +188,16 @@ func configureLogging() error { } func updateRegistryIndex(mustLoadIndex bool) error { + // Get release channel from config. + releaseChannel := getReleaseChannel(dataRoot) + + // Set indexes based on the release channel. + helper.SetIndexes(registry, releaseChannel) + + // Update registry to use pre-releases or not. + registry.SetUsePreReleases(releaseChannel != helper.ReleaseChannelStable) + + // Load indexes from disk or network, if needed and desired. err := registry.LoadIndexes(context.Background()) if err != nil { log.Printf("WARNING: error loading indexes: %s\n", err) @@ -218,6 +206,7 @@ func updateRegistryIndex(mustLoadIndex bool) error { } } + // Load versions from disk to know which others we have and which are available. err = registry.ScanStorage("") if err != nil { log.Printf("WARNING: error during storage scan: %s\n", err) @@ -247,13 +236,23 @@ func detectInstallationDir() string { return parent } -func stagingActive() bool { - // Check flag and env variable. - if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" { - return true +func getReleaseChannel(dataRoot *utils.DirStructure) string { + configData, err := ioutil.ReadFile(filepath.Join(dataRoot.Path, "config.json")) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("WARNING: failed to read config.json to get release channel: %s", err) + } + return helper.ReleaseChannelStable } - // Check if staging index is present and acessible. - _, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json")) - return err == nil + // Get release channel from config, validate and return it. + channel := gjson.GetBytes(configData, helper.ReleaseChannelJSONKey).String() + switch channel { + case helper.ReleaseChannelStable, + helper.ReleaseChannelBeta, + helper.ReleaseChannelStaging: + return channel + default: + return helper.ReleaseChannelStable + } } diff --git a/cmds/portmaster-start/update.go b/cmds/portmaster-start/update.go index 1b89b113..69c26e2c 100644 --- a/cmds/portmaster-start/update.go +++ b/cmds/portmaster-start/update.go @@ -40,8 +40,7 @@ var ( func indexRequired(cmd *cobra.Command) bool { switch cmd { - case updateCmd, - purgeCmd: + case updateCmd, purgeCmd: return true default: return false @@ -49,35 +48,9 @@ func indexRequired(cmd *cobra.Command) bool { } func downloadUpdates() error { - // mark required updates - if onWindows { - registry.MandatoryUpdates = []string{ - helper.PlatformIdentifier("core/portmaster-core.exe"), - helper.PlatformIdentifier("kext/portmaster-kext.dll"), - helper.PlatformIdentifier("kext/portmaster-kext.sys"), - helper.PlatformIdentifier("start/portmaster-start.exe"), - helper.PlatformIdentifier("notifier/portmaster-notifier.exe"), - helper.PlatformIdentifier("notifier/portmaster-snoretoast.exe"), - } - } else { - registry.MandatoryUpdates = []string{ - helper.PlatformIdentifier("core/portmaster-core"), - helper.PlatformIdentifier("start/portmaster-start"), - helper.PlatformIdentifier("notifier/portmaster-notifier"), - } - } - - // add updates that we require on all platforms. - registry.MandatoryUpdates = append( - registry.MandatoryUpdates, - helper.PlatformIdentifier("app/portmaster-app.zip"), - "all/ui/modules/portmaster.zip", - ) - - // Add assets that need unpacking. - registry.AutoUnpack = []string{ - helper.PlatformIdentifier("app/portmaster-app.zip"), - } + // Set required updates. + registry.MandatoryUpdates = helper.MandatoryUpdates() + registry.AutoUnpack = helper.AutoUnpackUpdates() // logging is configured as a persistent pre-run method inherited from // the root command but since we don't use run.Run() we need to start @@ -100,8 +73,8 @@ func downloadUpdates() error { return fmt.Errorf("failed to create update dir: %w", err) } - // Reset registry state. - registry.Reset() + // Reset registry resources. + registry.ResetResources() } // Update all indexes. diff --git a/go.mod b/go.mod index 9a0f387b..da6f30f1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b // indirect github.com/tevino/abool v1.2.0 - github.com/tidwall/gjson v1.7.5 // indirect + github.com/tidwall/gjson v1.7.5 github.com/tidwall/sjson v1.1.6 // indirect github.com/tjfoc/gmsm v1.4.0 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect From 417d7e4743e2b1412466d1214d079e9c01a99272 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 3 Jun 2021 23:30:27 +0200 Subject: [PATCH 4/7] Fix some linter errors --- cmds/portmaster-start/lock.go | 3 +- cmds/portmaster-start/logs.go | 2 +- cmds/portmaster-start/recover_linux.go | 4 +- cmds/portmaster-start/run.go | 14 ++-- cmds/portmaster-start/shutdown.go | 2 +- cmds/portmaster-start/version.go | 106 +++++++++++++------------ 6 files changed, 66 insertions(+), 65 deletions(-) diff --git a/cmds/portmaster-start/lock.go b/cmds/portmaster-start/lock.go index 1178ac53..07fe0988 100644 --- a/cmds/portmaster-start/lock.go +++ b/cmds/portmaster-start/lock.go @@ -14,7 +14,6 @@ import ( ) func checkAndCreateInstanceLock(path, name string) (pid int32, err error) { - lockFilePath := filepath.Join(dataRoot.Path, path, fmt.Sprintf("%s-lock.pid", name)) // read current pid file @@ -86,7 +85,7 @@ func createInstanceLock(lockFilePath string) error { } // create lock file - err = ioutil.WriteFile(lockFilePath, []byte(fmt.Sprintf("%d", os.Getpid())), 0666) + err = ioutil.WriteFile(lockFilePath, []byte(fmt.Sprintf("%d", os.Getpid())), 0666) //nolint:gosec if err != nil { return err } diff --git a/cmds/portmaster-start/logs.go b/cmds/portmaster-start/logs.go index 2e73f4ca..c982febb 100644 --- a/cmds/portmaster-start/logs.go +++ b/cmds/portmaster-start/logs.go @@ -96,7 +96,7 @@ func getPmStartLogFile(ext string) *os.File { }, info.Version(), ext) } -//nolint:deadcode,unused // false positive on linux, currently used by windows only +//nolint:unused // false positive on linux, currently used by windows only func logControlError(cErr error) { // check if error present if cErr == nil { diff --git a/cmds/portmaster-start/recover_linux.go b/cmds/portmaster-start/recover_linux.go index ecb6735a..06529431 100644 --- a/cmds/portmaster-start/recover_linux.go +++ b/cmds/portmaster-start/recover_linux.go @@ -19,8 +19,8 @@ var recoverIPTablesCmd = &cobra.Command{ // we don't get the errno of the actual error and need to parse the // output instead. Make sure it's always english by setting LC_ALL=C currentLocale := os.Getenv("LC_ALL") - os.Setenv("LC_ALL", "C") // nolint:errcheck - we tried at least ... - defer os.Setenv("LC_ALL", currentLocale) // nolint:errcheck + os.Setenv("LC_ALL", "C") + defer os.Setenv("LC_ALL", currentLocale) err := interception.DeactivateNfqueueFirewall() if err == nil { diff --git a/cmds/portmaster-start/run.go b/cmds/portmaster-start/run.go index f36e5d80..9564f7fa 100644 --- a/cmds/portmaster-start/run.go +++ b/cmds/portmaster-start/run.go @@ -33,17 +33,17 @@ var ( childIsRunning = abool.NewBool(false) ) -// Options for starting component +// Options for starting component. type Options struct { Name string Identifier string // component identifier + ShortIdentifier string // populated automatically LockPathPrefix string PIDFile bool - ShortIdentifier string // populated automatically - SuppressArgs bool // do not use any args - AllowDownload bool // allow download of component if it is not yet available - AllowHidingWindow bool // allow hiding the window of the subprocess - NoOutput bool // do not use stdout/err if logging to file is available (did not fail to open log file) + SuppressArgs bool // do not use any args + AllowDownload bool // allow download of component if it is not yet available + AllowHidingWindow bool // allow hiding the window of the subprocess + NoOutput bool // do not use stdout/err if logging to file is available (did not fail to open log file) } func init() { @@ -313,7 +313,7 @@ func execute(opts *Options, args []string) (cont bool, err error) { log.Printf("starting %s %s\n", binPath, strings.Join(args, " ")) // create command - exc := exec.Command(binPath, args...) //nolint:gosec // everything is okay + exc := exec.Command(binPath, args...) if !runningInConsole && opts.AllowHidingWindow { // Windows only: diff --git a/cmds/portmaster-start/shutdown.go b/cmds/portmaster-start/shutdown.go index b42218a1..f136f344 100644 --- a/cmds/portmaster-start/shutdown.go +++ b/cmds/portmaster-start/shutdown.go @@ -7,7 +7,7 @@ import ( var ( startupComplete = make(chan struct{}) // signal that the start procedure completed (is never closed, just signaled once) shuttingDown = make(chan struct{}) // signal that we are shutting down (will be closed, may not be closed directly, use initiateShutdown) - //nolint:deadcode,unused // false positive on linux, currently used by windows only + //nolint:unused // false positive on linux, currently used by windows only shutdownError error // protected by shutdownLock shutdownLock sync.Mutex ) diff --git a/cmds/portmaster-start/version.go b/cmds/portmaster-start/version.go index 9f3dc662..88af21e6 100644 --- a/cmds/portmaster-start/version.go +++ b/cmds/portmaster-start/version.go @@ -12,62 +12,64 @@ import ( "github.com/spf13/cobra" ) -var showShortVersion bool -var showAllVersions bool -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Display various portmaster versions", - Args: cobra.NoArgs, - PersistentPreRunE: func(*cobra.Command, []string) error { - if showAllVersions { - // If we are going to show all component versions, - // we need the registry to be configured. - if err := configureRegistry(false); err != nil { - return err - } - } - - return nil - }, - RunE: func(*cobra.Command, []string) error { - if !showAllVersions { - if showShortVersion { - fmt.Println(info.Version()) - return nil - } - - fmt.Println(info.FullVersion()) - return nil - } - - fmt.Printf("portmaster-start %s\n\n", info.Version()) - fmt.Printf("Assets:\n") - - all := registry.Export() - keys := make([]string, 0, len(all)) - for identifier := range all { - keys = append(keys, identifier) - } - sort.Strings(keys) - - tw := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - for _, identifier := range keys { - res := all[identifier] - - if showShortVersion { - // in "short" mode, skip all resources that are irrelevant on that platform - if !strings.HasPrefix(identifier, "all") && !strings.HasPrefix(identifier, runtime.GOOS) { - continue +var ( + showShortVersion bool + showAllVersions bool + versionCmd = &cobra.Command{ + Use: "version", + Short: "Display various portmaster versions", + Args: cobra.NoArgs, + PersistentPreRunE: func(*cobra.Command, []string) error { + if showAllVersions { + // If we are going to show all component versions, + // we need the registry to be configured. + if err := configureRegistry(false); err != nil { + return err } } - fmt.Fprintf(tw, " %s\t%s\n", identifier, res.SelectedVersion.VersionNumber) - } - tw.Flush() + return nil + }, + RunE: func(*cobra.Command, []string) error { + if !showAllVersions { + if showShortVersion { + fmt.Println(info.Version()) + return nil + } - return nil - }, -} + fmt.Println(info.FullVersion()) + return nil + } + + fmt.Printf("portmaster-start %s\n\n", info.Version()) + fmt.Printf("Assets:\n") + + all := registry.Export() + keys := make([]string, 0, len(all)) + for identifier := range all { + keys = append(keys, identifier) + } + sort.Strings(keys) + + tw := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + for _, identifier := range keys { + res := all[identifier] + + if showShortVersion { + // in "short" mode, skip all resources that are irrelevant on that platform + if !strings.HasPrefix(identifier, "all") && !strings.HasPrefix(identifier, runtime.GOOS) { + continue + } + } + + fmt.Fprintf(tw, " %s\t%s\n", identifier, res.SelectedVersion.VersionNumber) + } + tw.Flush() + + return nil + }, + } +) func init() { flags := versionCmd.Flags() From 9da5ff86d3731878b656f27fc0fcc3756cc2c938 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 3 Jun 2021 23:30:52 +0200 Subject: [PATCH 5/7] Adapt updatemgr to new release channel handling --- cmds/updatemgr/main.go | 20 +----- cmds/updatemgr/release.go | 145 ++++++++++++++++++++++++++------------ cmds/updatemgr/scan.go | 6 +- cmds/updatemgr/staging.go | 96 ------------------------- 4 files changed, 105 insertions(+), 162 deletions(-) delete mode 100644 cmds/updatemgr/staging.go diff --git a/cmds/updatemgr/main.go b/cmds/updatemgr/main.go index a0f969f4..267aade0 100644 --- a/cmds/updatemgr/main.go +++ b/cmds/updatemgr/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "os" "path/filepath" @@ -31,24 +30,7 @@ var rootCmd = &cobra.Command{ } registry = &updater.ResourceRegistry{} - err = registry.Initialize(utils.NewDirStructure(absDistPath, 0o755)) - if err != nil { - return err - } - - registry.AddIndex(updater.Index{ - Path: "stable.json", - Stable: true, - Beta: false, - }) - - registry.AddIndex(updater.Index{ - Path: "beta.json", - Stable: false, - Beta: true, - }) - - err = registry.LoadIndexes(context.TODO()) + err = registry.Initialize(utils.NewDirStructure(absDistPath, 0755)) if err != nil { return err } diff --git a/cmds/updatemgr/release.go b/cmds/updatemgr/release.go index 32a296a8..6579a21d 100644 --- a/cmds/updatemgr/release.go +++ b/cmds/updatemgr/release.go @@ -4,83 +4,140 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path/filepath" - "sort" "github.com/spf13/cobra" ) +var ( + releaseCmd = &cobra.Command{ + Use: "release", + Short: "Release scans the distribution directory and creates registry indexes and the symlink structure", + RunE: release, + } + preReleaseCmd = &cobra.Command{ + Use: "prerelease", + Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes", + Args: cobra.ExactArgs(1), + RunE: prerelease, + } + resetPreReleases bool + includeUnreleased bool +) + func init() { rootCmd.AddCommand(releaseCmd) -} - -var releaseCmd = &cobra.Command{ - Use: "release", - Short: "Release scans the distribution directory and creates registry indexes and the symlink structure", - RunE: release, + rootCmd.AddCommand(preReleaseCmd) + preReleaseCmd.Flags().BoolVar(&resetPreReleases, "reset", false, "Reset pre-release assets") } func release(cmd *cobra.Command, args []string) error { - // Set stable and beta to latest version. - updateToLatestVersion(true, true) + return writeIndex( + "stable", + getChannelVersions("", false), + ) +} - // Export versions. - betaData, err := json.MarshalIndent(exportSelected(true), "", " ") - if err != nil { - return err +func prerelease(cmd *cobra.Command, args []string) error { + channel := args[0] + + // Check if we want to reset instead. + if resetPreReleases { + return removeFilesFromIndex(getChannelVersions(channel, true)) } - stableData, err := json.MarshalIndent(exportSelected(false), "", " ") + + return writeIndex( + channel, + getChannelVersions(channel, false), + ) +} + +func writeIndex(channel string, versions map[string]string) error { + // Export versions and format them. + versionData, err := json.MarshalIndent(versions, "", " ") if err != nil { return err } - // Build destination paths. - betaIndexFilePath := filepath.Join(registry.StorageDir().Path, "beta.json") - stableIndexFilePath := filepath.Join(registry.StorageDir().Path, "stable.json") + // Build destination path. + indexFilePath := filepath.Join(registry.StorageDir().Path, channel+".json") - // Print previews. - fmt.Printf("beta (%s):\n", betaIndexFilePath) - fmt.Println(string(betaData)) - fmt.Printf("\nstable: (%s)\n", stableIndexFilePath) - fmt.Println(string(stableData)) + // Print preview. + fmt.Printf("%s (%s):\n", channel, indexFilePath) + fmt.Println(string(versionData)) // Ask for confirmation. - if !confirm("\nDo you want to write these new indexes (and update latest symlinks)?") { + if !confirm("\nDo you want to write this index?") { fmt.Println("aborted...") return nil } - // Write indexes. - err = ioutil.WriteFile(betaIndexFilePath, betaData, 0o644) //nolint:gosec // 0644 is intended + // Write new index to disk. + err = ioutil.WriteFile(indexFilePath, versionData, 0644) //nolint:gosec // 0644 is intended if err != nil { return err } - fmt.Printf("written %s\n", betaIndexFilePath) - - err = ioutil.WriteFile(stableIndexFilePath, stableData, 0o644) //nolint:gosec // 0644 is intended - if err != nil { - return err - } - fmt.Printf("written %s\n", stableIndexFilePath) - - // Create symlinks to latest stable versions. - symlinksDir := registry.StorageDir().ChildDir("latest", 0o755) - err = registry.CreateSymlinks(symlinksDir) - if err != nil { - return err - } - fmt.Printf("updated stable symlinks in %s\n", symlinksDir.Path) + fmt.Printf("written %s\n", indexFilePath) return nil } -func updateToLatestVersion(stable, beta bool) error { - for _, resource := range registry.Export() { - sort.Sort(resource) - err := resource.AddVersion(resource.Versions[0].VersionNumber, false, stable, beta) +func removeFilesFromIndex(versions map[string]string) error { + // Print preview. + fmt.Println("To be deleted:") + for _, filePath := range versions { + fmt.Println(filePath) + } + + // Ask for confirmation. + if !confirm("\nDo you want to delete these files?") { + fmt.Println("aborted...") + return nil + } + + // Delete files. + for _, filePath := range versions { + err := os.Remove(filePath) if err != nil { return err } } + + fmt.Println("deleted") return nil } + +func getChannelVersions(channel string, storagePath bool) map[string]string { + // Sort all versions. + registry.SelectVersions() + export := registry.Export() + + // Go through all versions and save the highest version, if not stable or beta. + versions := make(map[string]string) + for _, rv := range export { + for _, v := range rv.Versions { + // Ignore versions that don't match the release channel. + if v.SemVer().Prerelease() != channel { + // Stop at the first stable version, nothing should ever be selected + // beyond that. + if v.SemVer().Prerelease() == "" { + break + } + + continue + } + + // Add highest version of matching release channel. + if storagePath { + versions[rv.Identifier] = rv.GetFile().Path() + } else { + versions[rv.Identifier] = v.VersionNumber + } + + break + } + } + + return versions +} diff --git a/cmds/updatemgr/scan.go b/cmds/updatemgr/scan.go index afcd489c..ff9e0dce 100644 --- a/cmds/updatemgr/scan.go +++ b/cmds/updatemgr/scan.go @@ -19,7 +19,7 @@ var scanCmd = &cobra.Command{ func scan(cmd *cobra.Command, args []string) error { // Reset and rescan. - registry.Reset() + registry.ResetResources() err := registry.ScanStorage("") if err != nil { return err @@ -36,8 +36,8 @@ func scan(cmd *cobra.Command, args []string) error { return nil } -func exportSelected(beta bool) map[string]string { - registry.SetBeta(beta) +func exportSelected(preReleases bool) map[string]string { + registry.SetUsePreReleases(preReleases) registry.SelectVersions() export := registry.Export() diff --git a/cmds/updatemgr/staging.go b/cmds/updatemgr/staging.go deleted file mode 100644 index 63398b9b..00000000 --- a/cmds/updatemgr/staging.go +++ /dev/null @@ -1,96 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/spf13/cobra" -) - -var ( - stageReset bool -) - -func init() { - rootCmd.AddCommand(stageCmd) - stageCmd.Flags().BoolVar(&stageReset, "reset", false, "Reset staging assets") -} - -var stageCmd = &cobra.Command{ - Use: "stage", - Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes", - RunE: stage, -} - -func stage(cmd *cobra.Command, args []string) error { - // Check if we want to reset staging instead. - if stageReset { - for _, stagedPath := range exportStaging(true) { - err := os.Remove(stagedPath) - if err != nil { - return err - } - } - return nil - } - - // Export all staged versions and format them. - stagingData, err := json.MarshalIndent(exportStaging(false), "", " ") - if err != nil { - return err - } - - // Build destination path. - stagingIndexFilePath := filepath.Join(registry.StorageDir().Path, "staging.json") - - // Print preview. - fmt.Printf("staging (%s):\n", stagingIndexFilePath) - fmt.Println(string(stagingData)) - - // Ask for confirmation. - if !confirm("\nDo you want to write this index?") { - fmt.Println("aborted...") - return nil - } - - // Write new index to disk. - err = ioutil.WriteFile(stagingIndexFilePath, stagingData, 0o644) //nolint:gosec // 0644 is intended - if err != nil { - return err - } - fmt.Printf("written %s\n", stagingIndexFilePath) - - return nil -} - -func exportStaging(storagePath bool) map[string]string { - // Sort all versions. - registry.SetBeta(false) - registry.SelectVersions() - export := registry.Export() - - // Go through all versions and save the highest version, if not stable or beta. - versions := make(map[string]string) - for _, rv := range export { - // Get highest version. - v := rv.Versions[0] - - // Do not take stable or beta releases into account. - if v.StableRelease || v.BetaRelease { - continue - } - - // Add highest version to staging - if storagePath { - rv.SelectedVersion = v - versions[rv.Identifier] = rv.GetFile().Path() - } else { - versions[rv.Identifier] = v.VersionNumber - } - } - - return versions -} From ce192c724352b84600b96cbcb7b927626e8752c1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 7 Jun 2021 11:03:48 +0200 Subject: [PATCH 6/7] Add special release channel --- updates/config.go | 7 ++++++- updates/helper/indexes.go | 14 +++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/updates/config.go b/updates/config.go index 8043038b..c8dc2b33 100644 --- a/updates/config.go +++ b/updates/config.go @@ -49,9 +49,14 @@ func registerConfig() error { Description: "Production releases for testing new features that may break and cause interruption.", Value: helper.ReleaseChannelBeta, }, + { + Name: "Special", + Description: "Special releases or version changes for troubleshooting. Only use temporarily and when instructed.", + Value: helper.ReleaseChannelSpecial, + }, { Name: "Staging", - Description: "Development releases for testing random things and experimenting. Dangerous - only use when told so.", + Description: "Dangerous development releases for testing random things and experimenting. Only use temporarily and when instructed.", Value: helper.ReleaseChannelStaging, }, }, diff --git a/updates/helper/indexes.go b/updates/helper/indexes.go index 97689db4..49b26b63 100644 --- a/updates/helper/indexes.go +++ b/updates/helper/indexes.go @@ -10,6 +10,7 @@ const ( ReleaseChannelStable = "stable" ReleaseChannelBeta = "beta" ReleaseChannelStaging = "staging" + ReleaseChannelSpecial = "special" ) func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string) { @@ -21,14 +22,14 @@ func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string) { // Always add the stable index as a base. registry.AddIndex(updater.Index{ - Path: "stable.json", + Path: ReleaseChannelStable + ".json", }) // Add beta index if in beta or staging channel. if releaseChannel == ReleaseChannelBeta || releaseChannel == ReleaseChannelStaging { registry.AddIndex(updater.Index{ - Path: "beta.json", + Path: ReleaseChannelBeta + ".json", PreRelease: true, }) } @@ -36,11 +37,18 @@ func SetIndexes(registry *updater.ResourceRegistry, releaseChannel string) { // Add staging index if in staging channel. if releaseChannel == ReleaseChannelStaging { registry.AddIndex(updater.Index{ - Path: "staging.json", + Path: ReleaseChannelStaging + ".json", PreRelease: true, }) } + // Add special index if in special channel. + if releaseChannel == ReleaseChannelSpecial { + registry.AddIndex(updater.Index{ + Path: ReleaseChannelSpecial + ".json", + }) + } + // Add the intel index last, as it updates the fastest and should not be // crippled by other faulty indexes. It can only specify versions for its // scope anyway. From 27d0f23f41b65de93b8f623be0ba3f5145196869 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 7 Jun 2021 11:20:39 +0200 Subject: [PATCH 7/7] Implement review suggestions --- cmds/portmaster-start/lock.go | 1 + cmds/portmaster-start/logs.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmds/portmaster-start/lock.go b/cmds/portmaster-start/lock.go index 07fe0988..9635744d 100644 --- a/cmds/portmaster-start/lock.go +++ b/cmds/portmaster-start/lock.go @@ -85,6 +85,7 @@ func createInstanceLock(lockFilePath string) error { } // create lock file + // TODO: Investigate required permissions. err = ioutil.WriteFile(lockFilePath, []byte(fmt.Sprintf("%d", os.Getpid())), 0666) //nolint:gosec if err != nil { return err diff --git a/cmds/portmaster-start/logs.go b/cmds/portmaster-start/logs.go index c982febb..34f00867 100644 --- a/cmds/portmaster-start/logs.go +++ b/cmds/portmaster-start/logs.go @@ -96,7 +96,7 @@ func getPmStartLogFile(ext string) *os.File { }, info.Version(), ext) } -//nolint:unused // false positive on linux, currently used by windows only +//nolint:unused // false positive on linux, currently used by windows only. TODO: move to a _windows file. func logControlError(cErr error) { // check if error present if cErr == nil { @@ -112,7 +112,7 @@ func logControlError(cErr error) { fmt.Fprintln(errorFile, cErr.Error()) } -//nolint:deadcode,unused // false positive on linux, currently used by windows only +//nolint:deadcode,unused // false positive on linux, currently used by windows only. TODO: move to a _windows file. func runAndLogControlError(wrappedFunc func(cmd *cobra.Command, args []string) error) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { err := wrappedFunc(cmd, args)