diff --git a/cmds/updatemgr/main.go b/cmds/updatemgr/main.go index a157af07..1d998581 100644 --- a/cmds/updatemgr/main.go +++ b/cmds/updatemgr/main.go @@ -9,44 +9,54 @@ import ( "github.com/safing/portmaster/service/updates" ) -var binaryMap = map[string]updates.Artifact{ - "geoipv4.mmdb.gz": { - Filename: "geoipv4.mmdb", - Unpack: "gz", - }, - "geoipv6.mmdb.gz": { - Filename: "geoipv6.mmdb", - Unpack: "gz", - }, -} +var bundleSettings = updates.BundleFileSettings{ + Name: "Portmaster Binaries", + PrimaryArtifact: "linux_amd64/portmaster-core", + BaseURL: "https://updates.safing.io/", + IgnoreFiles: []string{ + // Indexes, checksums, latest symlinks. + "*.json", + "sha256*.txt", + "latest/**", -var ignoreFiles = map[string]struct{}{ - "bin-index.json": {}, - "intel-index.json": {}, + // Signatures. + "*.sig", + "**/*.sig", + + // Related, but not required artifacts. + "**/*.apk", + "**/*install*", + "**/spn-hub*", + "**/jess*", + "**/hubs*.json", + "**/*mini*.mmdb.gz", + + // Deprecated artifacts. + "**/profilemgr*.zip", + "**/settings*.zip", + "**/monitor*.zip", + "**/base*.zip", + "**/console*.zip", + "**/portmaster-wintoast*.dll", + "**/portmaster-snoretoast*.exe", + "**/portmaster-kext*.dll", + }, + UnpackFiles: map[string]string{ + "gz": "**/*.gz", + "zip": "**/app2/**/portmaster-app*.zip", + }, } func main() { dir := flag.String("dir", "", "path to the directory that contains the artifacts") - name := flag.String("name", "", "name of the bundle") - version := flag.String("version", "", "version of the bundle") flag.Parse() if *dir == "" { fmt.Fprintf(os.Stderr, "-dir parameter is required\n") return } - if *name == "" { - fmt.Fprintf(os.Stderr, "-name parameter is required\n") - return - } - settings := updates.BundleFileSettings{ - Name: *name, - Version: *version, - Properties: binaryMap, - IgnoreFiles: ignoreFiles, - } - bundle, err := updates.GenerateBundleFromDir(*dir, settings) + bundle, err := updates.GenerateBundleFromDir(*dir, bundleSettings) if err != nil { fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err) return diff --git a/go.mod b/go.mod index 436df094..9590ae56 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus v4.1.0+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect diff --git a/go.sum b/go.sum index 50df69d6..fbd91dd0 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4= github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= diff --git a/service/updates/bundle.go b/service/updates/bundle.go index 64ffc2a1..deee6666 100644 --- a/service/updates/bundle.go +++ b/service/updates/bundle.go @@ -24,6 +24,8 @@ type Artifact struct { Platform string `json:"Platform,omitempty"` Unpack string `json:"Unpack,omitempty"` Version string `json:"Version,omitempty"` + + localFile string } func (a *Artifact) GetFileMode() os.FileMode { diff --git a/service/updates/bundlegeneration.go b/service/updates/bundlegeneration.go index 3f82e367..b7117926 100644 --- a/service/updates/bundlegeneration.go +++ b/service/updates/bundlegeneration.go @@ -4,94 +4,281 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" + "io/fs" "os" "path" "path/filepath" "regexp" + "slices" "strings" "time" + "github.com/gobwas/glob" semver "github.com/hashicorp/go-version" ) type BundleFileSettings struct { - Name string - Version string - Properties map[string]Artifact - IgnoreFiles map[string]struct{} + Name string + Version string + PrimaryArtifact string + BaseURL string + + Templates map[string]Artifact + IgnoreFiles []string + UnpackFiles map[string]string + + cleanedBaseURL string + ignoreFilesGlobs []glob.Glob + unpackFilesGlobs map[string]glob.Glob +} + +func (bs *BundleFileSettings) init() error { + // Transform base URL into expected format. + bs.cleanedBaseURL = strings.TrimSuffix(bs.BaseURL, "/") + "/" + + // Parse ignore files patterns. + bs.ignoreFilesGlobs = make([]glob.Glob, 0, len(bs.IgnoreFiles)) + for _, pattern := range bs.IgnoreFiles { + g, err := glob.Compile(pattern, os.PathSeparator) + if err != nil { + return fmt.Errorf("invalid ingore files pattern %q: %w", pattern, err) + } + bs.ignoreFilesGlobs = append(bs.ignoreFilesGlobs, g) + } + + // Parse unpack files patterns. + bs.unpackFilesGlobs = make(map[string]glob.Glob) + for setting, pattern := range bs.UnpackFiles { + g, err := glob.Compile(pattern, os.PathSeparator) + if err != nil { + return fmt.Errorf("invalid unpack files pattern %q: %w", pattern, err) + } + bs.unpackFilesGlobs[setting] = g + } + + return nil +} + +// IsIgnored returns whether a filename should be ignored. +func (bs *BundleFileSettings) IsIgnored(filename string) bool { + for _, ignoreGlob := range bs.ignoreFilesGlobs { + if ignoreGlob.Match(filename) { + return true + } + } + + return false +} + +// UnpackSetting returns the unpack setings for the given filename. +func (bs *BundleFileSettings) UnpackSetting(filename string) (string, error) { + var foundSetting string + +settings: + for unpackSetting, matchGlob := range bs.unpackFilesGlobs { + switch { + case !matchGlob.Match(filename): + // Check next if glob does not match. + continue settings + case foundSetting == "": + // First find, save setting. + foundSetting = unpackSetting + case foundSetting != unpackSetting: + // Additional find, and setting is not the same. + return "", errors.New("matches contradicting unpack settings") + } + } + + return foundSetting, nil } // GenerateBundleFromDir generates a bundle from a given folder. func GenerateBundleFromDir(bundleDir string, settings BundleFileSettings) (*Bundle, error) { - bundleDirName := filepath.Base(bundleDir) + artifacts := make(map[string]Artifact) - artifacts := make([]Artifact, 0, 5) - err := filepath.Walk(bundleDir, func(path string, info os.FileInfo, err error) error { + // Initialize. + err := settings.init() + if err != nil { + return nil, fmt.Errorf("invalid bundle settings: %w", err) + } + bundleDir, err = filepath.Abs(bundleDir) + if err != nil { + return nil, fmt.Errorf("invalid bundle dir: %w", err) + } + + err = filepath.WalkDir(bundleDir, func(fullpath string, d fs.DirEntry, err error) error { + // Fail on access error. if err != nil { return err } - // Skip folders - if info.IsDir() { + + // Step 1: Extract information and check ignores. + + // Skip folders. + if d.IsDir() { return nil } - identifier, version, ok := getIdentifierAndVersion(info.Name()) - if !ok { - identifier = info.Name() + // Get relative path for processing. + relpath, err := filepath.Rel(bundleDir, fullpath) + if err != nil { + return fmt.Errorf("invalid relative path for %s: %w", fullpath, err) } // Check if file is in the ignore list. - if _, ok := settings.IgnoreFiles[identifier]; ok { + if settings.IsIgnored(relpath) { return nil } - artifact := Artifact{} - - // Check if the caller provided properties for the artifact. - if p, ok := settings.Properties[identifier]; ok { - artifact = p + // Extract version, if present. + identifier, version, ok := getIdentifierAndVersion(d.Name()) + if !ok { + // Fallback to using filename as identifier, which is normal for the simplified system. + identifier = d.Name() + version = "" + } + var versionNum *semver.Version + if version != "" { + versionNum, err = semver.NewVersion(version) + if err != nil { + return fmt.Errorf("invalid version %s for %s: %w", relpath, version, err) + } } - // Set filename of artifact if not set by the caller. + // Extract platform. + platform := "all" + before, _, found := strings.Cut(relpath, string(os.PathSeparator)) + if found { + platform = before + } + + // Step 2: Check and compare file version. + + // Make the key platform specific since there can be same filename for multiple platforms. + key := platform + "/" + identifier + existing, ok := artifacts[key] + if ok { + // Check for duplicates and mixed versioned/non-versioned. + switch { + case existing.Version == version: + return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath) + case (existing.Version == "") != (version == ""): + return fmt.Errorf("both a versioned and non-versioned file for: %s: %s and %s", key, existing.localFile, fullpath) + } + + // Compare versions. + existingVersion, _ := semver.NewVersion(existing.Version) + switch { + case existingVersion.Equal(versionNum): + return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath) + case existingVersion.GreaterThan(versionNum): + // New version is older, skip. + return nil + } + } + + // Step 3: Create new Artifact. + + artifact := Artifact{} + + // Check if the caller provided a template for the artifact. + if t, ok := settings.Templates[identifier]; ok { + artifact = t + } + + // Set artifact properties. if artifact.Filename == "" { artifact.Filename = identifier } - - artifact.Version = version - - // Fill the platform of the artifact - parentDir := filepath.Base(filepath.Dir(path)) - if parentDir != "all" && parentDir != bundleDirName { - artifact.Platform = parentDir + if len(artifact.URLs) == 0 && settings.BaseURL != "" { + artifact.URLs = []string{settings.cleanedBaseURL + relpath} + } + if artifact.Platform == "" { + artifact.Platform = platform + } + if artifact.Unpack == "" { + unpackSetting, err := settings.UnpackSetting(relpath) + if err != nil { + return fmt.Errorf("invalid unpack setting for %s at %s: %w", key, relpath, err) + } + artifact.Unpack = unpackSetting + } + if artifact.Version == "" { + artifact.Version = version } - // Fill the hash - hash, err := getSHA256(path, artifact.Unpack) - if err != nil { - return fmt.Errorf("failed to calculate hash of file: %s %w", path, err) - } - artifact.SHA256 = hash + // Set local file path. + artifact.localFile = fullpath - artifacts = append(artifacts, artifact) + // Save new artifact to map. + artifacts[key] = artifact return nil }) if err != nil { - return nil, fmt.Errorf("failed to walk the dir: %w", err) + return nil, fmt.Errorf("scanning dir: %w", err) } - // Filter artifact so we have single version for each file - artifacts, err = selectLatestArtifacts(artifacts) - if err != nil { - return nil, fmt.Errorf("failed to select artifact version: %w", err) - } - - return &Bundle{ + // Create base bundle. + bundle := &Bundle{ Name: settings.Name, Version: settings.Version, - Artifacts: artifacts, Published: time.Now(), - }, nil + } + if bundle.Version == "" && settings.PrimaryArtifact != "" { + pv, ok := artifacts[settings.PrimaryArtifact] + if ok { + bundle.Version = pv.Version + } + } + if bundle.Name == "" { + bundle.Name = strings.Trim(filepath.Base(bundleDir), "./\\") + } + + // Convert to slice and compute hashes. + export := make([]Artifact, 0, len(artifacts)) + for _, artifact := range artifacts { + // Compute hash. + hash, err := getSHA256(artifact.localFile, artifact.Unpack) + if err != nil { + return nil, fmt.Errorf("calculate hash of file: %s %w", artifact.localFile, err) + } + artifact.SHA256 = hash + + // Remove "all" platform IDs. + if artifact.Platform == "all" { + artifact.Platform = "" + } + + // Remove default versions. + if artifact.Version == bundle.Version { + artifact.Version = "" + } + + // Add to export slice. + export = append(export, artifact) + } + + // Sort final artifacts. + slices.SortFunc(export, func(a, b Artifact) int { + switch { + case a.Filename != b.Filename: + return strings.Compare(a.Filename, b.Filename) + case a.Platform != b.Platform: + return strings.Compare(a.Platform, b.Platform) + case a.Version != b.Version: + return strings.Compare(a.Version, b.Version) + case a.SHA256 != b.SHA256: + return strings.Compare(a.SHA256, b.SHA256) + default: + return 0 + } + }) + + // Assign and return. + bundle.Artifacts = export + return bundle, nil } func selectLatestArtifacts(artifacts []Artifact) ([]Artifact, error) {