github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[WIP] Improve bundle generation

Browse Source
This commit is contained in:
Daniel
2024-10-09 16:50:46 +02:00
parent 8b68243cc6
commit 5d9088f27e
5 changed files with 271 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
service/updates/bundle.go
View File

@@ -24,6 +24,8 @@ type Artifact struct {
Platform string `json:"Platform,omitempty"`
Unpack string `json:"Unpack,omitempty"`
Version string `json:"Version,omitempty"`
localFile string
}
func (a *Artifact) GetFileMode() os.FileMode {

273
service/updates/bundlegeneration.go
View File

@@ -4,94 +4,281 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
"github.com/gobwas/glob"
semver "github.com/hashicorp/go-version"
)
type BundleFileSettings struct {
Name string
Version string
Properties map[string]Artifact
IgnoreFiles map[string]struct{}
Name string
Version string
PrimaryArtifact string
BaseURL string
Templates map[string]Artifact
IgnoreFiles []string
UnpackFiles map[string]string
cleanedBaseURL string
ignoreFilesGlobs []glob.Glob
unpackFilesGlobs map[string]glob.Glob
}
func (bs *BundleFileSettings) init() error {
// Transform base URL into expected format.
bs.cleanedBaseURL = strings.TrimSuffix(bs.BaseURL, "/") + "/"
// Parse ignore files patterns.
bs.ignoreFilesGlobs = make([]glob.Glob, 0, len(bs.IgnoreFiles))
for _, pattern := range bs.IgnoreFiles {
g, err := glob.Compile(pattern, os.PathSeparator)
if err != nil {
return fmt.Errorf("invalid ingore files pattern %q: %w", pattern, err)
}
bs.ignoreFilesGlobs = append(bs.ignoreFilesGlobs, g)
}
// Parse unpack files patterns.
bs.unpackFilesGlobs = make(map[string]glob.Glob)
for setting, pattern := range bs.UnpackFiles {
g, err := glob.Compile(pattern, os.PathSeparator)
if err != nil {
return fmt.Errorf("invalid unpack files pattern %q: %w", pattern, err)
}
bs.unpackFilesGlobs[setting] = g
}
return nil
}
// IsIgnored returns whether a filename should be ignored.
func (bs *BundleFileSettings) IsIgnored(filename string) bool {
for _, ignoreGlob := range bs.ignoreFilesGlobs {
if ignoreGlob.Match(filename) {
return true
}
}
return false
}
// UnpackSetting returns the unpack setings for the given filename.
func (bs *BundleFileSettings) UnpackSetting(filename string) (string, error) {
var foundSetting string
settings:
for unpackSetting, matchGlob := range bs.unpackFilesGlobs {
switch {
case !matchGlob.Match(filename):
// Check next if glob does not match.
continue settings
case foundSetting == "":
// First find, save setting.
foundSetting = unpackSetting
case foundSetting != unpackSetting:
// Additional find, and setting is not the same.
return "", errors.New("matches contradicting unpack settings")
}
}
return foundSetting, nil
}
// GenerateBundleFromDir generates a bundle from a given folder.
func GenerateBundleFromDir(bundleDir string, settings BundleFileSettings) (*Bundle, error) {
bundleDirName := filepath.Base(bundleDir)
artifacts := make(map[string]Artifact)
artifacts := make([]Artifact, 0, 5)
err := filepath.Walk(bundleDir, func(path string, info os.FileInfo, err error) error {
// Initialize.
err := settings.init()
if err != nil {
return nil, fmt.Errorf("invalid bundle settings: %w", err)
}
bundleDir, err = filepath.Abs(bundleDir)
if err != nil {
return nil, fmt.Errorf("invalid bundle dir: %w", err)
}
err = filepath.WalkDir(bundleDir, func(fullpath string, d fs.DirEntry, err error) error {
// Fail on access error.
if err != nil {
return err
}
// Skip folders
if info.IsDir() {
// Step 1: Extract information and check ignores.
// Skip folders.
if d.IsDir() {
return nil
}
identifier, version, ok := getIdentifierAndVersion(info.Name())
if !ok {
identifier = info.Name()
// Get relative path for processing.
relpath, err := filepath.Rel(bundleDir, fullpath)
if err != nil {
return fmt.Errorf("invalid relative path for %s: %w", fullpath, err)
}
// Check if file is in the ignore list.
if _, ok := settings.IgnoreFiles[identifier]; ok {
if settings.IsIgnored(relpath) {
return nil
}
artifact := Artifact{}
// Check if the caller provided properties for the artifact.
if p, ok := settings.Properties[identifier]; ok {
artifact = p
// Extract version, if present.
identifier, version, ok := getIdentifierAndVersion(d.Name())
if !ok {
// Fallback to using filename as identifier, which is normal for the simplified system.
identifier = d.Name()
version = ""
}
var versionNum *semver.Version
if version != "" {
versionNum, err = semver.NewVersion(version)
if err != nil {
return fmt.Errorf("invalid version %s for %s: %w", relpath, version, err)
}
}
// Set filename of artifact if not set by the caller.
// Extract platform.
platform := "all"
before, _, found := strings.Cut(relpath, string(os.PathSeparator))
if found {
platform = before
}
// Step 2: Check and compare file version.
// Make the key platform specific since there can be same filename for multiple platforms.
key := platform + "/" + identifier
existing, ok := artifacts[key]
if ok {
// Check for duplicates and mixed versioned/non-versioned.
switch {
case existing.Version == version:
return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath)
case (existing.Version == "") != (version == ""):
return fmt.Errorf("both a versioned and non-versioned file for: %s: %s and %s", key, existing.localFile, fullpath)
}
// Compare versions.
existingVersion, _ := semver.NewVersion(existing.Version)
switch {
case existingVersion.Equal(versionNum):
return fmt.Errorf("duplicate version for %s: %s and %s", key, existing.localFile, fullpath)
case existingVersion.GreaterThan(versionNum):
// New version is older, skip.
return nil
}
}
// Step 3: Create new Artifact.
artifact := Artifact{}
// Check if the caller provided a template for the artifact.
if t, ok := settings.Templates[identifier]; ok {
artifact = t
}
// Set artifact properties.
if artifact.Filename == "" {
artifact.Filename = identifier
}
artifact.Version = version
// Fill the platform of the artifact
parentDir := filepath.Base(filepath.Dir(path))
if parentDir != "all" && parentDir != bundleDirName {
artifact.Platform = parentDir
if len(artifact.URLs) == 0 && settings.BaseURL != "" {
artifact.URLs = []string{settings.cleanedBaseURL + relpath}
}
if artifact.Platform == "" {
artifact.Platform = platform
}
if artifact.Unpack == "" {
unpackSetting, err := settings.UnpackSetting(relpath)
if err != nil {
return fmt.Errorf("invalid unpack setting for %s at %s: %w", key, relpath, err)
}
artifact.Unpack = unpackSetting
}
if artifact.Version == "" {
artifact.Version = version
}
// Fill the hash
hash, err := getSHA256(path, artifact.Unpack)
if err != nil {
return fmt.Errorf("failed to calculate hash of file: %s %w", path, err)
}
artifact.SHA256 = hash
// Set local file path.
artifact.localFile = fullpath
artifacts = append(artifacts, artifact)
// Save new artifact to map.
artifacts[key] = artifact
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to walk the dir: %w", err)
return nil, fmt.Errorf("scanning dir: %w", err)
}
// Filter artifact so we have single version for each file
artifacts, err = selectLatestArtifacts(artifacts)
if err != nil {
return nil, fmt.Errorf("failed to select artifact version: %w", err)
}
return &Bundle{
// Create base bundle.
bundle := &Bundle{
Name: settings.Name,
Version: settings.Version,
Artifacts: artifacts,
Published: time.Now(),
}, nil
}
if bundle.Version == "" && settings.PrimaryArtifact != "" {
pv, ok := artifacts[settings.PrimaryArtifact]
if ok {
bundle.Version = pv.Version
}
}
if bundle.Name == "" {
bundle.Name = strings.Trim(filepath.Base(bundleDir), "./\\")
}
// Convert to slice and compute hashes.
export := make([]Artifact, 0, len(artifacts))
for _, artifact := range artifacts {
// Compute hash.
hash, err := getSHA256(artifact.localFile, artifact.Unpack)
if err != nil {
return nil, fmt.Errorf("calculate hash of file: %s %w", artifact.localFile, err)
}
artifact.SHA256 = hash
// Remove "all" platform IDs.
if artifact.Platform == "all" {
artifact.Platform = ""
}
// Remove default versions.
if artifact.Version == bundle.Version {
artifact.Version = ""
}
// Add to export slice.
export = append(export, artifact)
}
// Sort final artifacts.
slices.SortFunc(export, func(a, b Artifact) int {
switch {
case a.Filename != b.Filename:
return strings.Compare(a.Filename, b.Filename)
case a.Platform != b.Platform:
return strings.Compare(a.Platform, b.Platform)
case a.Version != b.Version:
return strings.Compare(a.Version, b.Version)
case a.SHA256 != b.SHA256:
return strings.Compare(a.SHA256, b.SHA256)
default:
return 0
}
})
// Assign and return.
bundle.Artifacts = export
return bundle, nil
}
func selectLatestArtifacts(artifacts []Artifact) ([]Artifact, error) {