Improve updatemgr and updates module

cmds/updatemgr/convert.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/safing/portmaster/service/updates"
+)
+
+func convertV1(indexData []byte, baseURL string, lastUpdate time.Time) (*updates.Index, error) {
+	// Parse old index.
+	oldIndex := make(map[string]string)
+	err := json.Unmarshal(indexData, &oldIndex)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse old v1 index: %w", err)
+	}
+
+	// Create new index.
+	newIndex := &updates.Index{
+		Published: lastUpdate,
+		Artifacts: make([]*updates.Artifact, 0, len(oldIndex)),
+	}
+
+	// Convert all entries.
+	if err := convertEntries(newIndex, baseURL, oldIndex); err != nil {
+		return nil, err
+	}
+
+	return newIndex, nil
+}
+
+type IndexV2 struct {
+	Channel   string
+	Published time.Time
+	Releases  map[string]string
+}
+
+func convertV2(indexData []byte, baseURL string) (*updates.Index, error) {
+	// Parse old index.
+	oldIndex := &IndexV2{}
+	err := json.Unmarshal(indexData, oldIndex)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse old v2 index: %w", err)
+	}
+
+	// Create new index.
+	newIndex := &updates.Index{
+		Published: oldIndex.Published,
+		Artifacts: make([]*updates.Artifact, 0, len(oldIndex.Releases)),
+	}
+
+	// Convert all entries.
+	if err := convertEntries(newIndex, baseURL, oldIndex.Releases); err != nil {
+		return nil, err
+	}
+
+	return newIndex, nil
+}
+
+func convertEntries(index *updates.Index, baseURL string, entries map[string]string) error {
+entries:
+	for identifier, version := range entries {
+		dir, filename := path.Split(identifier)
+		artifactPath := GetVersionedPath(identifier, version)
+
+		// Check if file is to be ignored.
+		if scanConfig.IsIgnored(artifactPath) {
+			continue entries
+		}
+
+		// Get the platform.
+		var platform string
+		splittedPath := strings.Split(dir, "/")
+		if len(splittedPath) >= 1 {
+			platform = splittedPath[0]
+			if platform == "all" {
+				platform = ""
+			}
+		} else {
+			continue entries
+		}
+
+		// Create new artifact.
+		newArtifact := &updates.Artifact{
+			Filename: filename,
+			URLs:     []string{baseURL + artifactPath},
+			Platform: platform,
+			Version:  version,
+		}
+
+		// Derive unpack setting.
+		unpack, err := scanConfig.UnpackSetting(filename)
+		if err != nil {
+			return fmt.Errorf("failed to get unpack setting for %s: %w", filename, err)
+		}
+		newArtifact.Unpack = unpack
+
+		// Add to new index.
+		index.Artifacts = append(index.Artifacts, newArtifact)
+	}
+
+	return nil
+}
+
+// GetVersionedPath combines the identifier and version and returns it as a file path.
+func GetVersionedPath(identifier, version string) (versionedPath string) {
+	identifierPath, filename := path.Split(identifier)
+
+	// Split the filename where the version should go.
+	splittedFilename := strings.SplitN(filename, ".", 2)
+	// Replace `.` with `-` for the filename format.
+	transformedVersion := strings.Replace(version, ".", "-", 2)
+
+	// Put everything back together and return it.
+	versionedPath = identifierPath + splittedFilename[0] + "_v" + transformedVersion
+	if len(splittedFilename) > 1 {
+		versionedPath += "." + splittedFilename[1]
+	}
+	return versionedPath
+}

cmds/updatemgr/download.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/spf13/cobra"
+
+	"github.com/safing/portmaster/base/log"
+	"github.com/safing/portmaster/service/updates"
+)
+
+const currentPlatform = runtime.GOOS + "_" + runtime.GOARCH
+
+var (
+	downloadCmd = &cobra.Command{
+		Use:   "download [index URL] [download dir]",
+		Short: "Download all artifacts by an index to a directory",
+		RunE:  download,
+		Args:  cobra.ExactArgs(2),
+	}
+
+	downloadPlatform string
+)
+
+func init() {
+	rootCmd.AddCommand(downloadCmd)
+	downloadCmd.Flags().StringVarP(&downloadPlatform, "platform", "p", currentPlatform, "Define platform to download artifacts for")
+}
+
+func download(cmd *cobra.Command, args []string) error {
+	// Args.
+	indexURL := args[0]
+	targetDir := args[1]
+
+	// Check target dir.
+	stat, err := os.Stat(targetDir)
+	if err != nil {
+		return fmt.Errorf("failed to access target dir: %w", err)
+	}
+	if !stat.IsDir() {
+		return errors.New("target is not a directory")
+	}
+
+	// Create temporary directories.
+	tmpDownload, err := os.MkdirTemp("", "portmaster-updatemgr-download-")
+	if err != nil {
+		return err
+	}
+	tmpPurge, err := os.MkdirTemp("", "portmaster-updatemgr-purge-")
+	if err != nil {
+		return err
+	}
+
+	// Create updater.
+	u, err := updates.New(nil, "", updates.Config{
+		Name:              "Downloader",
+		Directory:         targetDir,
+		DownloadDirectory: tmpDownload,
+		PurgeDirectory:    tmpPurge,
+		IndexURLs:         []string{indexURL},
+		IndexFile:         "index.json",
+		Platform:          downloadPlatform,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Start logging.
+	err = log.Start(log.InfoLevel.Name(), true, "")
+	if err != nil {
+		return err
+	}
+
+	// Download artifacts.
+	err = u.ForceUpdate()
+
+	// Stop logging.
+	log.Shutdown()
+
+	// Remove tmp dirs
+	os.RemoveAll(tmpDownload)
+	os.RemoveAll(tmpPurge)
+
+	return err
+}

cmds/updatemgr/mirror.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/safing/portmaster/service/updates"
+)
+
+var (
+	// UserAgent is an HTTP User-Agent that is used to add
+	// more context to requests made by the registry when
+	// fetching resources from the update server.
+	UserAgent = fmt.Sprintf("Portmaster Update Mgr (%s %s)", runtime.GOOS, runtime.GOARCH)
+
+	client http.Client
+)
+
+func init() {
+	rootCmd.AddCommand(mirrorCmd)
+}
+
+var (
+	mirrorCmd = &cobra.Command{
+		Use:   "mirror [index URL] [mirror dir]",
+		Short: "Mirror all artifacts by an index to a directory, keeping the directory structure and file names intact",
+		RunE:  mirror,
+		Args:  cobra.ExactArgs(2),
+	}
+)
+
+func mirror(cmd *cobra.Command, args []string) error {
+	// Args.
+	indexURL := args[0]
+	targetDir := args[1]
+
+	// Check target dir.
+	stat, err := os.Stat(targetDir)
+	if err != nil {
+		return fmt.Errorf("failed to access target dir: %w", err)
+	}
+	if !stat.IsDir() {
+		return errors.New("target is not a directory")
+	}
+
+	// Calculate Base URL.
+	u, err := url.Parse(indexURL)
+	if err != nil {
+		return fmt.Errorf("invalid index URL: %w", err)
+	}
+	indexPath := u.Path
+	u.RawQuery = ""
+	u.RawFragment = ""
+	u.Path = ""
+	u.RawPath = ""
+	baseURL := u.String() + "/"
+
+	// Download Index.
+	fmt.Println("downloading index...")
+	indexData, err := downloadData(cmd.Context(), indexURL)
+	if err != nil {
+		return fmt.Errorf("download index: %w", err)
+	}
+
+	// Parse (and convert) index.
+	var index *updates.Index
+	_, newIndexName := path.Split(indexPath)
+	switch {
+	case strings.HasSuffix(indexPath, ".v3.json"):
+		index = &updates.Index{}
+		err := json.Unmarshal(indexData, index)
+		if err != nil {
+			return fmt.Errorf("parse v3 index: %w", err)
+		}
+	case strings.HasSuffix(indexPath, ".v2.json"):
+		index, err = convertV2(indexData, baseURL)
+		if err != nil {
+			return fmt.Errorf("convert v2 index: %w", err)
+		}
+		newIndexName = strings.TrimSuffix(newIndexName, ".v2.json") + ".v3.json"
+	case strings.HasSuffix(indexPath, ".json"):
+		index, err = convertV1(indexData, baseURL, time.Now())
+		if err != nil {
+			return fmt.Errorf("convert v1 index: %w", err)
+		}
+		newIndexName = strings.TrimSuffix(newIndexName, ".json") + ".v3.json"
+	default:
+		return errors.New("invalid index file extension")
+	}
+
+	// Download and save artifacts.
+	for _, artifact := range index.Artifacts {
+		fmt.Printf("downloading %s...\n", artifact.Filename)
+
+		// Download artifact and add any missing checksums.
+		artifactData, artifactLocation, err := getArtifact(cmd.Context(), artifact)
+		if err != nil {
+			return fmt.Errorf("get artifact %s: %w", artifact.Filename, err)
+		}
+
+		// Write artifact to correct location.
+		artifactDst := filepath.Join(targetDir, filepath.FromSlash(artifactLocation))
+		artifactDir, _ := filepath.Split(artifactDst)
+		err = os.MkdirAll(artifactDir, 0o0755)
+		if err != nil {
+			return fmt.Errorf("create artifact dir %s: %w", artifactDir, err)
+		}
+		err = os.WriteFile(artifactDst, artifactData, 0o0644)
+		if err != nil {
+			return fmt.Errorf("save artifact %s: %w", artifact.Filename, err)
+		}
+	}
+
+	// Save index.
+	indexJson, err := json.MarshalIndent(index, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal index: %w", err)
+	}
+	indexDst := filepath.Join(targetDir, newIndexName)
+	err = os.WriteFile(indexDst, indexJson, 0o0644)
+	if err != nil {
+		return fmt.Errorf("write index to %s: %w", indexDst, err)
+	}
+
+	return err
+}
+
+func getArtifact(ctx context.Context, artifact *updates.Artifact) (artifactData []byte, artifactLocation string, err error) {
+	// Check URL.
+	if len(artifact.URLs) == 0 {
+		return nil, "", errors.New("no URLs defined")
+	}
+	u, err := url.Parse(artifact.URLs[0])
+	if err != nil {
+		return nil, "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	// Download data from URL.
+	artifactData, err = downloadData(ctx, artifact.URLs[0])
+	if err != nil {
+		return nil, "", fmt.Errorf("GET artifact: %w", err)
+	}
+
+	// Decompress artifact data, if configured.
+	var finalArtifactData []byte
+	if artifact.Unpack != "" {
+		finalArtifactData, err = updates.Decompress(artifact.Unpack, artifactData)
+		if err != nil {
+			return nil, "", fmt.Errorf("decompress: %w", err)
+		}
+	} else {
+		finalArtifactData = artifactData
+	}
+
+	// Verify or generate checksum.
+	if artifact.SHA256 != "" {
+		if err := updates.CheckSHA256Sum(finalArtifactData, artifact.SHA256); err != nil {
+			return nil, "", err
+		}
+	} else {
+		fileHash := sha256.New()
+		if _, err := io.Copy(fileHash, bytes.NewReader(finalArtifactData)); err != nil {
+			return nil, "", fmt.Errorf("digest file: %w", err)
+		}
+		artifact.SHA256 = hex.EncodeToString(fileHash.Sum(nil))
+	}
+
+	return artifactData, u.Path, nil
+}
+
+func downloadData(ctx context.Context, url string) ([]byte, error) {
+	// Setup request.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create GET request to %s: %w", url, err)
+	}
+	if UserAgent != "" {
+		req.Header.Set("User-Agent", UserAgent)
+	}
+
+	// Start request with shared http client.
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed a get file request to: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	// Check for HTTP status errors.
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("server returned non-OK status: %d %s", resp.StatusCode, resp.Status)
+	}
+
+	// Read the full body and return it.
+	content, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read body of response: %w", err)
+	}
+	return content, nil
+}

cmds/updatemgr/sign.go

@@ -69,7 +69,7 @@ func sign(cmd *cobra.Command, args []string) error {
 	}
 
 	// Parse index and check if it is valid.
-	index, err := updates.ParseIndex(unsignedIndexData, nil)
+	index, err := updates.ParseIndex(unsignedIndexData, "", nil)
 	if err != nil {
 		return fmt.Errorf("invalid index: %w", err)
 	}
@@ -85,7 +85,7 @@ func sign(cmd *cobra.Command, args []string) error {
 	}
 
 	// Check by parsing again.
-	index, err = updates.ParseIndex(signedIndexData, nil)
+	index, err = updates.ParseIndex(signedIndexData, "", nil)
 	if err != nil {
 		return fmt.Errorf("invalid index after signing: %w", err)
 	}