github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[WIP] Improve bundle generation

Browse Source
This commit is contained in:
Daniel
2024-10-09 16:50:46 +02:00
parent 8b68243cc6
commit 5d9088f27e
5 changed files with 271 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
cmds/updatemgr/main.go
View File

@@ -9,44 +9,54 @@ import (
"github.com/safing/portmaster/service/updates"
)
var binaryMap = map[string]updates.Artifact{
"geoipv4.mmdb.gz": {
Filename: "geoipv4.mmdb",
Unpack: "gz",
},
"geoipv6.mmdb.gz": {
Filename: "geoipv6.mmdb",
Unpack: "gz",
},
}
var bundleSettings = updates.BundleFileSettings{
Name: "Portmaster Binaries",
PrimaryArtifact: "linux_amd64/portmaster-core",
BaseURL: "https://updates.safing.io/",
IgnoreFiles: []string{
// Indexes, checksums, latest symlinks.
"*.json",
"sha256*.txt",
"latest/**",
var ignoreFiles = map[string]struct{}{
"bin-index.json": {},
"intel-index.json": {},
// Signatures.
"*.sig",
"**/*.sig",
// Related, but not required artifacts.
"**/*.apk",
"**/*install*",
"**/spn-hub*",
"**/jess*",
"**/hubs*.json",
"**/*mini*.mmdb.gz",
// Deprecated artifacts.
"**/profilemgr*.zip",
"**/settings*.zip",
"**/monitor*.zip",
"**/base*.zip",
"**/console*.zip",
"**/portmaster-wintoast*.dll",
"**/portmaster-snoretoast*.exe",
"**/portmaster-kext*.dll",
},
UnpackFiles: map[string]string{
"gz": "**/*.gz",
"zip": "**/app2/**/portmaster-app*.zip",
},
}
func main() {
dir := flag.String("dir", "", "path to the directory that contains the artifacts")
name := flag.String("name", "", "name of the bundle")
version := flag.String("version", "", "version of the bundle")
flag.Parse()
if *dir == "" {
fmt.Fprintf(os.Stderr, "-dir parameter is required\n")
return
}
if *name == "" {
fmt.Fprintf(os.Stderr, "-name parameter is required\n")
return
}
settings := updates.BundleFileSettings{
Name: *name,
Version: *version,
Properties: binaryMap,
IgnoreFiles: ignoreFiles,
}
bundle, err := updates.GenerateBundleFromDir(*dir, settings)
bundle, err := updates.GenerateBundleFromDir(*dir, bundleSettings)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err)
return

1
go.mod
View File

@@ -79,6 +79,7 @@ require (
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/godbus/dbus v4.1.0+incompatible // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect

2
go.sum
View File

@@ -85,6 +85,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW
github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4=
github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=

2
service/updates/bundle.go
View File

@@ -24,6 +24,8 @@ type Artifact struct {
Platform string `json:"Platform,omitempty"`
Unpack string `json:"Unpack,omitempty"`
Version string `json:"Version,omitempty"`
localFile string
}
func (a *Artifact) GetFileMode() os.FileMode {

273
service/updates/bundlegeneration.go
View File

@@ -4,94 +4,281 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
"github.com/gobwas/glob"
semver "github.com/hashicorp/go-version"
)
type BundleFileSettings struct {
Name string
Version string
Properties map[string]Artifact
IgnoreFiles map[string]struct{}
Name string
Version string
PrimaryArtifact string
BaseURL string
Templates map[string]Artifact
IgnoreFiles []string
UnpackFiles map[string]string
cleanedBaseURL string
ignoreFilesGlobs []glob.Glob
unpackFilesGlobs map[string]glob.Glob
}
func (bs *BundleFileSettings) init() error {
// Transform base URL into expected format.
bs.cleanedBaseURL = strings.TrimSuffix(bs.BaseURL, "/") + "/"
// Parse ignore files patterns.
bs.ignoreFilesGlobs = make([]glob.Glob, 0, len(bs.IgnoreFiles))
for _, pattern := range bs.IgnoreFiles {
g, err := glob.Compile(pattern, os.PathSeparator)
if err != nil {
return fmt.Errorf("invalid ingore files pattern %q: %w", pattern, err)
}
bs.ignoreFilesGlobs = append(bs.ignoreFilesGlobs, g)
}
// Parse unpack files patterns.
bs.unpackFilesGlobs = make(map[string]glob.Glob)
for setting, pattern := range bs.UnpackFiles {
g, err := glob.Compile(pattern, os.PathSeparator)
if err != nil {
return fmt.Errorf("invalid unpack files pattern %q: %w", pattern, err)
}
bs.unpackFilesGlobs[setting] = g
}
return nil
}
// IsIgnored returns whether a filename should be ignored.
func (bs *BundleFileSettings) IsIgnored(filename string) bool {
for _, ignoreGlob := range bs.ignoreFilesGlobs {
if ignoreGlob.Match(filename) {
return true
}
}
return false
}
// UnpackSetting returns the unpack setings for the given filename.
func (bs *BundleFileSettings) UnpackSetting(filename string) (string, error) {
var foundSetting string
settings:
for unpackSetting, matchGlob := range bs.unpackFilesGlobs {
switch {
case !matchGlob.Match(filename):
// Check next if glob does not match.
continue settings
case foundSetting == "":
// First find, save setting.
foundSetting = unpackSetting
case foundSetting != unpackSetting:
// Additional find, and setting is not the same.
return "", errors.New("matches contradicting unpack settings")
}
}
return foundSetting, nil
}
// GenerateBundleFromDir generates a bundle from a given folder.
func GenerateBundleFromDir(bundleDir string, settings BundleFileSettings) (*Bundle, error) {
bundleDirName := filepath.Base(bundleDir)
artifacts := make(map[string]Artifact)
artifacts := make([]Artifact, 0, 5)
err := filepath.Walk(bundleDir, func(path string, info os.FileInfo, err error) error {
// Initialize.
err := settings.init()
if err != nil {
return nil, fmt.Errorf("invalid bundle settings: %w", err)
}
bundleDir, err = filepath.Abs(bundleDir)
if err != nil {
return nil, fmt.Errorf("invalid bundle dir: %w", err)
}
err = filepath.WalkDir(bundleDir, func(fullpath string, d fs.DirEntry, err error) error {
// Fail on access error.
if err != nil {
return err
}
// Skip folders
if info.IsDir() {
// Step 1: Extract information and check ignores.
// Skip folders.
if d.IsDir() {
return nil
}
identifier, version, ok := getIdentifierAndVersion(info.Name())
if !ok {
identifier = info.Name()
// Get relative path for processing.
relpath, err := filepath.Rel(bundleDir, fullpath)
if err != nil {
return fmt.Errorf("invalid relative path for %s: %w", fullpath, err)
}
// Check if file is in the ignore list.
if _, ok := settings.IgnoreFiles[identifier]; ok {
if settings.IsIgnored(relpath) {
return nil
}
artifact := Artifact{}
// Check if the caller provided properties for the artifact.
if p, ok := settings.Properties[identifier]; ok {
artifact = p
// Extract version, if present.
identifier, version, ok := getIdentifierAndVersion(d.Name())
if !ok {
// Fallback to using filename as identifier, which is normal for the simplified system.
identifier = d.Name()
version = ""
}
var versionNum *semver.Version
if version != "" {
versionNum, err = semver.NewVersion(version)
if err != nil {
return fmt.Errorf("invalid version %s for %s: %w", relpath, version, err)
}
}
// Set filename of artifact if not set by the caller.
// Extract platform.
platform := "all"
before, _, found := strings.Cut(relpath, string(os.PathSeparator))
if found {
platform = before
}
// Step 2: Check and compare file version.
// Make the key platform specific since there can be same filename for multiple platforms.
key := platform + "/" + identifier
existing, ok := artifacts[key]
if ok {
// Check for duplicates and mixed versioned/non-versioned.
switch {
case existing.Version == version:
return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath)
case (existing.Version == "") != (version == ""):
return fmt.Errorf("both a versioned and non-versioned file for: %s: %s and %s", key, existing.localFile, fullpath)
}
// Compare versions.
existingVersion, _ := semver.NewVersion(existing.Version)
switch {
case existingVersion.Equal(versionNum):
return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath)
case existingVersion.GreaterThan(versionNum):
// New version is older, skip.
return nil
}
}
// Step 3: Create new Artifact.
artifact := Artifact{}
// Check if the caller provided a template for the artifact.
if t, ok := settings.Templates[identifier]; ok {
artifact = t
}
// Set artifact properties.
if artifact.Filename == "" {
artifact.Filename = identifier
}
artifact.Version = version
// Fill the platform of the artifact
parentDir := filepath.Base(filepath.Dir(path))
if parentDir != "all" && parentDir != bundleDirName {
artifact.Platform = parentDir
if len(artifact.URLs) == 0 && settings.BaseURL != "" {
artifact.URLs = []string{settings.cleanedBaseURL + relpath}
}
if artifact.Platform == "" {
artifact.Platform = platform
}
if artifact.Unpack == "" {
unpackSetting, err := settings.UnpackSetting(relpath)
if err != nil {
return fmt.Errorf("invalid unpack setting for %s at %s: %w", key, relpath, err)
}
artifact.Unpack = unpackSetting
}
if artifact.Version == "" {
artifact.Version = version
}
// Fill the hash
hash, err := getSHA256(path, artifact.Unpack)
if err != nil {
return fmt.Errorf("failed to calculate hash of file: %s %w", path, err)
}
artifact.SHA256 = hash
// Set local file path.
artifact.localFile = fullpath
artifacts = append(artifacts, artifact)
// Save new artifact to map.
artifacts[key] = artifact
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to walk the dir: %w", err)
return nil, fmt.Errorf("scanning dir: %w", err)
}
// Filter artifact so we have single version for each file
artifacts, err = selectLatestArtifacts(artifacts)
if err != nil {
return nil, fmt.Errorf("failed to select artifact version: %w", err)
}
return &Bundle{
// Create base bundle.
bundle := &Bundle{
Name: settings.Name,
Version: settings.Version,
Artifacts: artifacts,
Published: time.Now(),
}, nil
}
if bundle.Version == "" && settings.PrimaryArtifact != "" {
pv, ok := artifacts[settings.PrimaryArtifact]
if ok {
bundle.Version = pv.Version
}
}
if bundle.Name == "" {
bundle.Name = strings.Trim(filepath.Base(bundleDir), "./\\")
}
// Convert to slice and compute hashes.
export := make([]Artifact, 0, len(artifacts))
for _, artifact := range artifacts {
// Compute hash.
hash, err := getSHA256(artifact.localFile, artifact.Unpack)
if err != nil {
return nil, fmt.Errorf("calculate hash of file: %s %w", artifact.localFile, err)
}
artifact.SHA256 = hash
// Remove "all" platform IDs.
if artifact.Platform == "all" {
artifact.Platform = ""
}
// Remove default versions.
if artifact.Version == bundle.Version {
artifact.Version = ""
}
// Add to export slice.
export = append(export, artifact)
}
// Sort final artifacts.
slices.SortFunc(export, func(a, b Artifact) int {
switch {
case a.Filename != b.Filename:
return strings.Compare(a.Filename, b.Filename)
case a.Platform != b.Platform:
return strings.Compare(a.Platform, b.Platform)
case a.Version != b.Version:
return strings.Compare(a.Version, b.Version)
case a.SHA256 != b.SHA256:
return strings.Compare(a.SHA256, b.SHA256)
default:
return 0
}
})
// Assign and return.
bundle.Artifacts = export
return bundle, nil
}
func selectLatestArtifacts(artifacts []Artifact) ([]Artifact, error) {