Add updates module and fix issues

updates/doc.go

@@ -0,0 +1,9 @@
+package updates
+
+// current paths:
+// all/ui/assets.zip
+// all/ui/modules/base.zip
+// all/ui/modules/settings.zip
+// all/ui/modules/profilemgr.zip
+// all/ui/modules/monitor.zip
+// linux_amd64/app/portmaster-ui

updates/fetch.go

@@ -0,0 +1,122 @@
+package updates
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/google/renameio"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	updateURLs = []string{
+		"https://updates.safing.io",
+	}
+)
+
+func fetchFile(realFilepath, updateFilepath string, tries int) error {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], updateFilepath)
+	if err != nil {
+		return fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], updateFilepath, err)
+	}
+
+	// create destination dir
+	dirPath := filepath.Dir(realFilepath)
+	err = os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		return fmt.Errorf("updates: could not create updates folder: %s", dirPath)
+	}
+
+	// open file for writing
+	atomicFile, err := renameio.TempFile(filepath.Join(updateStoragePath, "tmp"), realFilepath)
+	if err != nil {
+		return fmt.Errorf("updates: could not create temp file for download: %s", err)
+	}
+	defer atomicFile.Cleanup()
+
+	// start file download
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		return fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	n, err := io.Copy(atomicFile, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength)
+	}
+
+	// finalize file
+	err = atomicFile.CloseAtomicallyReplace()
+	if err != nil {
+		return fmt.Errorf("updates: failed to finalize file %s: %s", realFilepath, err)
+	}
+	// set permissions
+	err = os.Chmod(realFilepath, 0644)
+	if err != nil {
+		log.Warningf("updates: failed to set permissions on downloaded file %s: %s", realFilepath, err)
+	}
+
+	log.Infof("update: fetched %s (stored to %s)", downloadURL, realFilepath)
+	return nil
+}
+
+func fetchData(downloadPath string, tries int) ([]byte, error) {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], downloadPath)
+	if err != nil {
+		return nil, fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], downloadPath, err)
+	}
+
+	// start file download
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
+	n, err := io.Copy(buf, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return nil, fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength)
+	}
+
+	return buf.Bytes(), nil
+}
+
+func joinURLandPath(baseURL, urlPath string) (string, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return "", err
+	}
+
+	u.Path = path.Join(u.Path, urlPath)
+	return u.String(), nil
+}

updates/file.go

@@ -0,0 +1,46 @@
+package updates
+
+// File represents a file from the update system.
+type File struct {
+	filepath string
+	version  string
+	stable   bool
+}
+
+func newFile(filepath string, version string, stable bool) *File {
+	return &File{
+		filepath: filepath,
+		version:  version,
+		stable:   stable,
+	}
+}
+
+// Path returns the filepath of the file.
+func (f *File) Path() string {
+	return f.filepath
+}
+
+// Version returns the version of the file.
+func (f *File) Version() string {
+	return f.version
+}
+
+// Stable returns whether the file is from a stable release.
+func (f *File) Stable() bool {
+	return f.stable
+}
+
+// Open opens the file and returns the
+func (f *File) Open() {
+
+}
+
+// ReportError reports an error back to Safing. This will not automatically blacklist the file.
+func (f *File) ReportError() {
+
+}
+
+// Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on.
+func (f *File) Blacklist() {
+
+}

updates/filename.go

@@ -0,0 +1,41 @@
+package updates
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var versionRegex = regexp.MustCompile("_v[0-9]+-[0-9]+-[0-9]+b?")
+
+func getIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) {
+	// extract version
+	rawVersion := versionRegex.FindString(versionedPath)
+	if rawVersion == "" {
+		return "", "", false
+	}
+
+	// replace - with . and trim _
+	version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", -1)
+
+	// put together without version
+	i := strings.Index(versionedPath, rawVersion)
+	if i < 0 {
+		// extracted version not in string (impossible)
+		return "", "", false
+	}
+	return versionedPath[:i] + versionedPath[i+len(rawVersion):], version, true
+}
+
+func getVersionedPath(identifier, version string) (versionedPath string) {
+	// split in half
+	splittedFilePath := strings.SplitN(identifier, ".", 2)
+	// replace . with -
+	transformedVersion := strings.Replace(version, ".", "-", -1)
+
+	// put together
+	if len(splittedFilePath) == 1 {
+		return fmt.Sprintf("%s_v%s", splittedFilePath[0], transformedVersion)
+	}
+	return fmt.Sprintf("%s_v%s.%s", splittedFilePath[0], transformedVersion, splittedFilePath[1])
+}

updates/get.go

@@ -0,0 +1,77 @@
+package updates
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	ErrNotFound = errors.New("the requested file could not be found")
+)
+
+// GetPlatformFile returns the latest platform specific file identified by the given identifier.
+func GetPlatformFile(identifier string) (*File, error) {
+	identifier = filepath.Join(fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), identifier)
+	// From https://golang.org/pkg/runtime/#GOARCH
+	// GOOS is the running program's operating system target: one of darwin, freebsd, linux, and so on.
+	// GOARCH is the running program's architecture target: one of 386, amd64, arm, s390x, and so on.
+	return loadOrFetchFile(identifier)
+}
+
+// GetFile returns the latest generic file identified by the given identifier.
+func GetFile(identifier string) (*File, error) {
+	identifier = filepath.Join("all", identifier)
+	return loadOrFetchFile(identifier)
+}
+
+func getLatestFilePath(identifier string) (versionedFilePath, version string, stable bool, ok bool) {
+	updatesLock.RLock()
+	version, ok = stableUpdates[identifier]
+	if !ok {
+		version, ok = latestUpdates[identifier]
+		if !ok {
+			log.Tracef("updates: file %s does not exist", identifier)
+			return "", "", false, false
+			// TODO: if in development mode, reload latest index to check for newly sideloaded updates
+			// err := reloadLatest()
+		}
+	}
+	updatesLock.RUnlock()
+
+	// TODO: Fix for stable release
+	return getVersionedPath(identifier, version), version, false, true
+}
+
+func loadOrFetchFile(identifier string) (*File, error) {
+	versionedFilePath, version, stable, ok := getLatestFilePath(identifier)
+	if !ok {
+		// TODO: if in development mode, search updates dir for sideloaded apps
+		return nil, ErrNotFound
+	}
+
+	// build final filepath
+	realFilePath := filepath.Join(updateStoragePath, versionedFilePath)
+	if _, err := os.Stat(realFilePath); err == nil {
+		// file exists
+		return newFile(realFilePath, version, stable), nil
+	}
+
+	// download file
+	log.Tracef("updates: starting download of %s", versionedFilePath)
+	var err error
+	for tries := 0; tries < 5; tries++ {
+		err := fetchFile(realFilePath, versionedFilePath, tries)
+		if err != nil {
+			log.Tracef("updates: failed to download %s: %s, retrying (%d)", versionedFilePath, err, tries+1)
+		} else {
+			return newFile(realFilePath, version, stable), nil
+		}
+	}
+	log.Warningf("updates: failed to download %s: %s", versionedFilePath, err)
+	return nil, err
+}

updates/get_test.go

@@ -0,0 +1,24 @@
+package updates
+
+import "testing"
+
+func testBuildVersionedFilePath(t *testing.T, identifier, version, expectedVersionedFilePath string) {
+	updatesLock.Lock()
+	stableUpdates[identifier] = version
+	// betaUpdates[identifier] = version
+	updatesLock.Unlock()
+
+	versionedFilePath, _, _, ok := getLatestFilePath(identifier)
+	if !ok {
+		t.Errorf("identifier %s should exist", identifier)
+	}
+	if versionedFilePath != expectedVersionedFilePath {
+		t.Errorf("unexpected versionedFilePath: %s", versionedFilePath)
+	}
+}
+
+func TestBuildVersionedFilePath(t *testing.T) {
+	testBuildVersionedFilePath(t, "path/to/asset.zip", "1.2.3", "path/to/asset_v1-2-3.zip")
+	testBuildVersionedFilePath(t, "path/to/asset.tar.gz", "1.2.3b", "path/to/asset_v1-2-3b.tar.gz")
+	testBuildVersionedFilePath(t, "path/to/asset", "1.2.3b", "path/to/asset_v1-2-3b")
+}

updates/latest.go

@@ -0,0 +1,141 @@
+package updates
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	stableUpdates = make(map[string]string)
+	betaUpdates   = make(map[string]string)
+	latestUpdates = make(map[string]string)
+	updatesLock   sync.RWMutex
+)
+
+// ReloadLatest reloads available updates from disk.
+func ReloadLatest() error {
+	newLatestUpdates := make(map[string]string)
+
+	// all
+	new, err1 := ScanForLatest(filepath.Join(updateStoragePath, "all"), false)
+	for key, val := range new {
+		newLatestUpdates[key] = val
+	}
+
+	// os_platform
+	new, err2 := ScanForLatest(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)), false)
+	for key, val := range new {
+		newLatestUpdates[key] = val
+	}
+
+	if err1 != nil && err2 != nil {
+		return fmt.Errorf("could not load latest update versions: %s, %s", err1, err2)
+	}
+
+	log.Tracef("updates: loading latest updates:")
+
+	for key, val := range newLatestUpdates {
+		log.Tracef("updates: %s v%s", key, val)
+	}
+
+	updatesLock.Lock()
+	latestUpdates = newLatestUpdates
+	updatesLock.Unlock()
+
+	log.Tracef("updates: load complete")
+
+	if len(stableUpdates) == 0 {
+		err := loadIndexesFromDisk()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ScanForLatest(baseDir string, hardFail bool) (latest map[string]string, lastError error) {
+	var added int
+	latest = make(map[string]string)
+
+	filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			lastError = err
+			if hardFail {
+				return err
+			}
+			log.Warningf("updates: could not read %s", path)
+			return nil
+		}
+		if !info.IsDir() {
+			added++
+		}
+
+		relativePath := strings.TrimLeft(strings.TrimPrefix(path, baseDir), "/")
+		identifierPath, version, ok := getIdentifierAndVersion(relativePath)
+		if !ok {
+			return nil
+		}
+
+		// add/update index
+		storedVersion, ok := latest[identifierPath]
+		if ok {
+			// FIXME: this will fail on multi-digit version segments!
+			if version > storedVersion {
+				latest[identifierPath] = version
+			}
+		} else {
+			latest[identifierPath] = version
+		}
+
+		return nil
+	})
+
+	if lastError != nil {
+		if hardFail {
+			return nil, lastError
+		}
+		if added == 0 {
+			return latest, lastError
+		}
+	}
+	return latest, nil
+}
+
+func loadIndexesFromDisk() error {
+	data, err := ioutil.ReadFile(filepath.Join(updateStoragePath, "stable.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Infof("updates: stable.json does not yet exist, waiting for first update cycle")
+			return nil
+		}
+		return err
+	}
+
+	newStableUpdates := make(map[string]string)
+	err = json.Unmarshal(data, &newStableUpdates)
+	if err != nil {
+		return err
+	}
+
+	if len(newStableUpdates) == 0 {
+		return errors.New("stable.json is empty")
+	}
+
+	log.Tracef("updates: loaded stable.json")
+
+	updatesLock.Lock()
+	stableUpdates = newStableUpdates
+	updatesLock.Unlock()
+
+	return nil
+}

updates/latest_test.go

@@ -0,0 +1,73 @@
+package updates
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, expectedVersion string) {
+	fullPath := filepath.Join(basePath, filePath)
+
+	// create dir
+	dirPath := filepath.Dir(fullPath)
+	err := os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+
+	// touch file
+	err = ioutil.WriteFile(fullPath, []byte{}, 0644)
+	if err != nil {
+		t.Fatalf("could not create test file: %s\n", err)
+		return
+	}
+
+	// run loadLatestScope
+	latest, err := ScanForLatest(basePath, true)
+	if err != nil {
+		t.Errorf("could not update latest: %s\n", err)
+		return
+	}
+	for key, val := range latest {
+		latestUpdates[key] = val
+	}
+
+	// test result
+	version, ok := latestUpdates[expectedIdentifier]
+	if !ok {
+		t.Errorf("identifier %s not in map", expectedIdentifier)
+		t.Errorf("current map: %v", latestUpdates)
+	}
+	if version != expectedVersion {
+		t.Errorf("unexpected version for %s: %s", filePath, version)
+	}
+}
+
+func TestLoadLatestScope(t *testing.T) {
+
+	updatesLock.Lock()
+	defer updatesLock.Unlock()
+
+	tmpDir, err := ioutil.TempDir("", "testing_")
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+	defer os.RemoveAll(tmpDir)
+
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3b.zip", "all/ui/assets.zip", "1.2.3b")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "1.2.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "1.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v2-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v2-1-1", "os_platform/portmaster/portmaster", "2.1.1")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "2.1.1")
+
+}

updates/main.go

@@ -0,0 +1,95 @@
+package updates
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Safing/portbase/database"
+	"github.com/Safing/portbase/modules"
+)
+
+var (
+	updateStoragePath string
+)
+
+func init() {
+	modules.Register("updates", prep, start, nil, "database")
+}
+
+func prep() error {
+	updateStoragePath = filepath.Join(database.GetDatabaseRoot(), "updates")
+
+	err := checkUpdateDirs()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func start() error {
+	err := ReloadLatest()
+	if err != nil {
+		return err
+	}
+
+	go updater()
+	return nil
+}
+
+func stop() error {
+	return os.RemoveAll(filepath.Join(updateStoragePath, "tmp"))
+}
+
+func checkUpdateDirs() error {
+	// all
+	err := checkDir(filepath.Join(updateStoragePath, "all"))
+	if err != nil {
+		return err
+	}
+
+	// os_platform
+	err = checkDir(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)))
+	if err != nil {
+		return err
+	}
+
+	// tmp
+	err = checkDir(filepath.Join(updateStoragePath, "tmp"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkDir(dirPath string) error {
+	f, err := os.Stat(dirPath)
+	if err == nil {
+		// file exists
+		if f.IsDir() {
+			return nil
+		}
+		err = os.Remove(dirPath)
+		if err != nil {
+			return fmt.Errorf("could not remove file %s to place dir: %s", dirPath, err)
+		}
+		err = os.MkdirAll(dirPath, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create dir %s: %s", dirPath, err)
+		}
+		return nil
+	}
+	// file does not exist
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(dirPath, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create dir %s: %s", dirPath, err)
+		}
+		return nil
+	}
+	// other error
+	return fmt.Errorf("failed to access %s: %s", dirPath, err)
+}

updates/updater.go

@@ -0,0 +1,88 @@
+package updates
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	"github.com/Safing/portbase/log"
+)
+
+func updater() {
+	time.Sleep(10 * time.Second)
+	for {
+		err := checkForUpdates()
+		if err != nil {
+			log.Warningf("updates: failed to check for updates: %s", err)
+		}
+		time.Sleep(1 * time.Hour)
+	}
+}
+
+func checkForUpdates() error {
+
+	// download new index
+	var data []byte
+	var err error
+	for tries := 0; tries < 3; tries++ {
+		data, err = fetchData("stable.json", tries)
+		if err == nil {
+			break
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	newStableUpdates := make(map[string]string)
+	err = json.Unmarshal(data, &newStableUpdates)
+	if err != nil {
+		return err
+	}
+
+	if len(newStableUpdates) == 0 {
+		return errors.New("stable.json is empty")
+	}
+
+	// FIXINSTABLE: correct log line
+	log.Infof("updates: downloaded new update index: stable.json (alpha until we actually reach stable)")
+
+	// update existing files
+	log.Tracef("updates: updating existing files")
+	updatesLock.RLock()
+	for identifier, newVersion := range newStableUpdates {
+		oldVersion, ok := latestUpdates[identifier]
+		if ok && newVersion != oldVersion {
+
+			filePath := getVersionedPath(identifier, newVersion)
+			realFilePath := filepath.Join(updateStoragePath, filePath)
+			for tries := 0; tries < 3; tries++ {
+				err := fetchFile(realFilePath, filePath, tries)
+				if err == nil {
+					break
+				}
+			}
+			if err != nil {
+				log.Warningf("failed to update %s to %s: %s", identifier, newVersion, err)
+			}
+
+		}
+	}
+	updatesLock.RUnlock()
+	log.Tracef("updates: finished updating existing files")
+
+	// update stable index
+	updatesLock.Lock()
+	stableUpdates = newStableUpdates
+	updatesLock.Unlock()
+
+	// save stable index
+	err = ioutil.WriteFile(filepath.Join(updateStoragePath, "stable.json"), data, 0644)
+	if err != nil {
+		log.Warningf("updates: failed to save new version of stable.json: %s", err)
+	}
+
+	return nil
+}

updates/uptool/.gitignore

@@ -0,0 +1 @@
+uptool

updates/uptool/root.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "uptool",
+	Short: "helper tool for the update process",
+	Run: func(cmd *cobra.Command, args []string) {
+		cmd.Usage()
+	},
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

updates/uptool/scan.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Safing/portmaster/updates"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(scanCmd)
+}
+
+var scanCmd = &cobra.Command{
+	Use:   "scan",
+	Short: "Scan the current directory and print the result",
+	RunE:  scan,
+}
+
+func scan(cmd *cobra.Command, args []string) error {
+
+	latest, err := updates.ScanForLatest(".", true)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.MarshalIndent(latest, "", " ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(string(data))
+	return nil
+}