diff --git a/firewall/master.go b/firewall/master.go index 61394bef..bc96c6f0 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -1,6 +1,7 @@ package firewall import ( + "fmt" "os" "strings" @@ -57,14 +58,14 @@ func DecideOnConnectionBeforeIntel(connection *network.Connection, fqdn string) } // check domain list - permitted, ok := profileSet.CheckDomain(fqdn) + permitted, reason, ok := profileSet.CheckEndpoint(fqdn, 0, 0, false) if ok { if permitted { - log.Infof("firewall: accepting connection %s, domain is whitelisted", connection) - connection.Accept("domain is whitelisted") + log.Infof("firewall: accepting connection %s, endpoint is whitelisted: %s", connection, reason) + connection.Accept(fmt.Sprintf("endpoint is whitelisted: %s", reason)) } else { - log.Infof("firewall: denying connection %s, domain is blacklisted", connection) - connection.Deny("domain is blacklisted") + log.Infof("firewall: denying connection %s, endpoint is blacklisted", connection) + connection.Deny("endpoint is blacklisted") } return } @@ -207,6 +208,8 @@ func DecideOnConnection(connection *network.Connection, pkt packet.Packet) { connection.Deny("peer to peer connections (to an IP) not allowed") return } + default: + } // check network scope @@ -269,6 +272,13 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet // Profile.ConnectPorts // Profile.ListenPorts + // grant self + if connection.Process().Pid == os.Getpid() { + log.Infof("firewall: granting own link %s", connection) + connection.Accept("") + return + } + // check if there is a profile profileSet := connection.Process().ProfileSet() if profileSet == nil { @@ -278,7 +288,18 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet } profileSet.Update(status.CurrentSecurityLevel()) - // get remote Port + // get host + var domainOrIP string + switch { + case strings.HasSuffix(connection.Domain, "."): + domainOrIP = connection.Domain + case connection.Direction: + domainOrIP = pkt.GetIPHeader().Src.String() + default: + domainOrIP = pkt.GetIPHeader().Dst.String() + } + + // get protocol / destination port protocol := pkt.GetIPHeader().Protocol var dstPort uint16 tcpUDPHeader := pkt.GetTCPUDPHeader() @@ -286,12 +307,12 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet dstPort = tcpUDPHeader.DstPort } - // check port list - permitted, ok := profileSet.CheckPort(connection.Direction, uint8(protocol), dstPort) + // check endpoints list + permitted, reason, ok := profileSet.CheckEndpoint(domainOrIP, uint8(protocol), dstPort, connection.Direction) if ok { if permitted { - log.Infof("firewall: accepting link %s", link) - link.Accept("port whitelisted") + log.Infof("firewall: accepting link %s, endpoint is whitelisted: %s", link, reason) + link.Accept(fmt.Sprintf("port whitelisted: %s", reason)) } else { log.Infof("firewall: denying link %s: port %d is blacklisted", link, dstPort) link.Deny("port blacklisted") @@ -301,16 +322,16 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet switch profileSet.GetProfileMode() { case profile.Whitelist: - log.Infof("firewall: denying link %s: port %d is not whitelisted", link, dstPort) - link.Deny("port is not whitelisted") + log.Infof("firewall: denying link %s: endpoint %d is not whitelisted", link, dstPort) + link.Deny("endpoint is not whitelisted") return case profile.Prompt: - log.Infof("firewall: accepting link %s: port %d is blacklisted", link, dstPort) - link.Accept("port permitted (prompting is not yet implemented)") + log.Infof("firewall: accepting link %s: endpoint %d is blacklisted", link, dstPort) + link.Accept("endpoint permitted (prompting is not yet implemented)") return case profile.Blacklist: - log.Infof("firewall: accepting link %s: port %d is not blacklisted", link, dstPort) - link.Accept("port is not blacklisted") + log.Infof("firewall: accepting link %s: endpoint %d is not blacklisted", link, dstPort) + link.Accept("endpoint is not blacklisted") return } diff --git a/main.go b/main.go index 4c359384..7f04bbb9 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( _ "github.com/Safing/portbase/database/storage/badger" _ "github.com/Safing/portmaster/firewall" _ "github.com/Safing/portmaster/nameserver" + _ "github.com/Safing/portmaster/ui" ) var ( diff --git a/process/database.go b/process/database.go index a28ef40d..3cb87eed 100644 --- a/process/database.go +++ b/process/database.go @@ -68,7 +68,7 @@ func (p *Process) Save() { // Delete deletes a process from the storage and propagates the change. func (p *Process) Delete() { p.Lock() - defer p.Lock() + defer p.Unlock() processesLock.Lock() delete(processes, p.Pid) @@ -79,7 +79,10 @@ func (p *Process) Delete() { go dbController.PushUpdate(p) } - profile.DeactivateProfileSet(p.profileSet) + // TODO: this should not be necessary, as processes should always have a profileSet. + if p.profileSet != nil { + profile.DeactivateProfileSet(p.profileSet) + } } // CleanProcessStorage cleans the storage from old processes. diff --git a/profile/endpoints.go b/profile/endpoints.go index 9e86eaf1..3716f824 100644 --- a/profile/endpoints.go +++ b/profile/endpoints.go @@ -56,8 +56,10 @@ func (e Endpoints) Check(domainOrIP string, protocol uint8, port uint16, checkRe isDomain := strings.HasSuffix(domainOrIP, ".") for _, entry := range e { - if ok, reason := entry.Matches(domainOrIP, protocol, port, isDomain, cachedGetDomainOfIP); ok { - return entry.Permit, reason, true + if entry != nil { + if ok, reason := entry.Matches(domainOrIP, protocol, port, isDomain, cachedGetDomainOfIP); ok { + return entry.Permit, reason, true + } } } diff --git a/profile/profile.go b/profile/profile.go index ba411079..5be0d1a8 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -57,6 +57,7 @@ func New() *Profile { } } +// MakeProfileKey creates the correct key for a profile with the given namespace and ID. func MakeProfileKey(namespace, ID string) string { return fmt.Sprintf("core:profiles/%s/%s", namespace, ID) } diff --git a/profile/specialprofiles.go b/profile/specialprofiles.go index 2226934e..d2e5ceb6 100644 --- a/profile/specialprofiles.go +++ b/profile/specialprofiles.go @@ -47,13 +47,15 @@ func getSpecialProfile(ID string) (*Profile, error) { func ensureServiceEndpointsDenyAll(p *Profile) (changed bool) { for _, ep := range p.ServiceEndpoints { - if ep.DomainOrIP == "" && - ep.Wildcard == true && - ep.Protocol == 0 && - ep.StartPort == 0 && - ep.EndPort == 0 && - ep.Permit == false { - return false + if ep != nil { + if ep.DomainOrIP == "" && + ep.Wildcard == true && + ep.Protocol == 0 && + ep.StartPort == 0 && + ep.EndPort == 0 && + ep.Permit == false { + return false + } } } diff --git a/ui/launch.go b/ui/launch.go new file mode 100644 index 00000000..e0f4086b --- /dev/null +++ b/ui/launch.go @@ -0,0 +1,66 @@ +package ui + +import ( + "errors" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + + "github.com/Safing/portbase/modules" + "github.com/Safing/portmaster/updates" +) + +var ( + launchUI bool +) + +func init() { + flag.BoolVar(&launchUI, "ui", false, "launch user interface and exit") +} + +func launchUIByFlag() error { + if !launchUI { + return nil + } + + err := updates.ReloadLatest() + if err != nil { + return err + } + + osAndPlatform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) + + switch osAndPlatform { + case "linux_amd64": + + file, err := updates.GetPlatformFile("app/portmaster-ui") + if err != nil { + return fmt.Errorf("ui currently not available: %s - you may need to first start portmaster and wait for it to fetch the update index", err) + } + + // check permission + info, err := os.Stat(file.Path()) + if info.Mode() != 0755 { + fmt.Printf("%v\n", info.Mode()) + err := os.Chmod(file.Path(), 0755) + if err != nil { + return fmt.Errorf("failed to set exec permissions on %s: %s", file.Path(), err) + } + } + + // exec + cmd := exec.Command(file.Path()) + err = cmd.Start() + if err != nil { + return fmt.Errorf("failed to start ui: %s", err) + } + + // gracefully exit portmaster + return modules.ErrCleanExit + + default: + return errors.New("this os/platform is no UI support yet") + } +} diff --git a/ui/module.go b/ui/module.go index 2b02f192..16acf9b1 100644 --- a/ui/module.go +++ b/ui/module.go @@ -5,13 +5,14 @@ import ( ) func init() { - modules.Register("ui", prep, start, stop, "database", "api") + modules.Register("ui", prep, nil, nil, "updates", "api") } func prep() error { - return nil -} + err := launchUIByFlag() + if err != nil { + return err + } -func stop() error { - return nil + return registerRoutes() } diff --git a/ui/serve.go b/ui/serve.go index 0d03d612..d6ef8af6 100644 --- a/ui/serve.go +++ b/ui/serve.go @@ -6,16 +6,16 @@ import ( "mime" "net/http" "net/url" - "path" "path/filepath" + "strings" "sync" resources "github.com/cookieo9/resources-go" "github.com/gorilla/mux" "github.com/Safing/portbase/api" - "github.com/Safing/portbase/database" "github.com/Safing/portbase/log" + "github.com/Safing/portmaster/updates" ) var ( @@ -25,66 +25,80 @@ var ( assetsLock sync.RWMutex ) -func start() error { - basePath := path.Join(database.GetDatabaseRoot(), "updates", "files", "apps") - - serveUIRouter := mux.NewRouter() - serveUIRouter.HandleFunc("/assets/{resPath:[a-zA-Z0-9/\\._-]+}", ServeAssets(basePath)) - serveUIRouter.HandleFunc("/app/{appName:[a-z]+}/", ServeApps(basePath)) - serveUIRouter.HandleFunc("/app/{appName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", ServeApps(basePath)) - serveUIRouter.HandleFunc("/", RedirectToControl) - - api.RegisterAdditionalRoute("/assets/", serveUIRouter) - api.RegisterAdditionalRoute("/app/", serveUIRouter) - api.RegisterAdditionalRoute("/", serveUIRouter) +func registerRoutes() error { + api.RegisterHandleFunc("/assets/{resPath:[a-zA-Z0-9/\\._-]+}", ServeBundle("assets")).Methods("GET", "HEAD") + api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}", redirAddSlash).Methods("GET", "HEAD") + api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/", ServeBundle("")).Methods("GET", "HEAD") + api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", ServeBundle("")).Methods("GET", "HEAD") + api.RegisterHandleFunc("/", RedirectToBase) return nil } -// ServeApps serves app files. -func ServeApps(basePath string) func(w http.ResponseWriter, r *http.Request) { +// ServeBundle serves bundles. +func ServeBundle(defaultModuleName string) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { + // log.Tracef("ui: request for %s", r.RequestURI) + vars := mux.Vars(r) - appName, ok := vars["appName"] + moduleName, ok := vars["moduleName"] if !ok { - http.Error(w, "missing app name", http.StatusBadRequest) - return + moduleName = defaultModuleName + if moduleName == "" { + http.Error(w, "missing module name", http.StatusBadRequest) + return + } } resPath, ok := vars["resPath"] - if !ok { - http.Error(w, "missing resource path", http.StatusBadRequest) - return + if !ok || strings.HasSuffix(resPath, "/") { + resPath = "index.html" } appsLock.RLock() - bundle, ok := apps[appName] + bundle, ok := apps[moduleName] appsLock.RUnlock() if ok { - ServeFileFromBundle(w, r, bundle, resPath) + ServeFileFromBundle(w, r, moduleName, bundle, resPath) return } - newBundle, err := resources.OpenZip(path.Join(basePath, fmt.Sprintf("%s.zip", appName))) + // get file from update system + zipFile, err := updates.GetFile(fmt.Sprintf("ui/modules/%s.zip", moduleName)) if err != nil { + if err == updates.ErrNotFound { + log.Tracef("ui: requested module %s does not exist", moduleName) + http.Error(w, err.Error(), http.StatusNotFound) + } else { + log.Tracef("ui: error loading module %s: %s", moduleName, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // open bundle + newBundle, err := resources.OpenZip(zipFile.Path()) + if err != nil { + log.Tracef("ui: error prepping module %s: %s", moduleName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } bundle = &resources.BundleSequence{newBundle} appsLock.Lock() - apps[appName] = bundle + apps[moduleName] = bundle appsLock.Unlock() - ServeFileFromBundle(w, r, bundle, resPath) + ServeFileFromBundle(w, r, moduleName, bundle, resPath) } } // ServeFileFromBundle serves a file from the given bundle. -func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundle *resources.BundleSequence, path string) { +func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName string, bundle *resources.BundleSequence, path string) { readCloser, err := bundle.Open(path) if err != nil { + log.Tracef("ui: error opening module %s: %s", bundleName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -110,45 +124,16 @@ func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundle *resourc return } -// ServeAssets serves global UI assets. -func ServeAssets(basePath string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - - vars := mux.Vars(r) - resPath, ok := vars["resPath"] - if !ok { - http.Error(w, "missing resource path", http.StatusBadRequest) - return - } - - assetsLock.RLock() - bundle := assets - assetsLock.RUnlock() - if bundle != nil { - ServeFileFromBundle(w, r, bundle, resPath) - } - - newBundle, err := resources.OpenZip(path.Join(basePath, "assets.zip")) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - bundle = &resources.BundleSequence{newBundle} - assetsLock.Lock() - assets = bundle - assetsLock.Unlock() - - ServeFileFromBundle(w, r, bundle, resPath) - } -} - -// RedirectToControl redirects the requests to the control app -func RedirectToControl(w http.ResponseWriter, r *http.Request) { - u, err := url.Parse("/app/control") +// RedirectToBase redirects the requests to the control app +func RedirectToBase(w http.ResponseWriter, r *http.Request) { + u, err := url.Parse("/ui/modules/base/") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } http.Redirect(w, r, r.URL.ResolveReference(u).String(), http.StatusPermanentRedirect) } + +func redirAddSlash(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, r.RequestURI+"/", http.StatusPermanentRedirect) +} diff --git a/updates/doc.go b/updates/doc.go new file mode 100644 index 00000000..4dd1c7c3 --- /dev/null +++ b/updates/doc.go @@ -0,0 +1,9 @@ +package updates + +// current paths: +// all/ui/assets.zip +// all/ui/modules/base.zip +// all/ui/modules/settings.zip +// all/ui/modules/profilemgr.zip +// all/ui/modules/monitor.zip +// linux_amd64/app/portmaster-ui diff --git a/updates/fetch.go b/updates/fetch.go new file mode 100644 index 00000000..9330d0c5 --- /dev/null +++ b/updates/fetch.go @@ -0,0 +1,122 @@ +package updates + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "time" + + "github.com/google/renameio" + + "github.com/Safing/portbase/log" +) + +var ( + updateURLs = []string{ + "https://updates.safing.io", + } +) + +func fetchFile(realFilepath, updateFilepath string, tries int) error { + // backoff when retrying + if tries > 0 { + time.Sleep(time.Duration(tries*tries) * time.Second) + } + + // create URL + downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], updateFilepath) + if err != nil { + return fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], updateFilepath, err) + } + + // create destination dir + dirPath := filepath.Dir(realFilepath) + err = os.MkdirAll(dirPath, 0755) + if err != nil { + return fmt.Errorf("updates: could not create updates folder: %s", dirPath) + } + + // open file for writing + atomicFile, err := renameio.TempFile(filepath.Join(updateStoragePath, "tmp"), realFilepath) + if err != nil { + return fmt.Errorf("updates: could not create temp file for download: %s", err) + } + defer atomicFile.Cleanup() + + // start file download + resp, err := http.Get(downloadURL) + if err != nil { + return fmt.Errorf("error fetching url (%s): %s", downloadURL, err) + } + defer resp.Body.Close() + + // download and write file + n, err := io.Copy(atomicFile, resp.Body) + if err != nil { + return fmt.Errorf("failed downloading %s: %s", downloadURL, err) + } + if resp.ContentLength != n { + return fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength) + } + + // finalize file + err = atomicFile.CloseAtomicallyReplace() + if err != nil { + return fmt.Errorf("updates: failed to finalize file %s: %s", realFilepath, err) + } + // set permissions + err = os.Chmod(realFilepath, 0644) + if err != nil { + log.Warningf("updates: failed to set permissions on downloaded file %s: %s", realFilepath, err) + } + + log.Infof("update: fetched %s (stored to %s)", downloadURL, realFilepath) + return nil +} + +func fetchData(downloadPath string, tries int) ([]byte, error) { + // backoff when retrying + if tries > 0 { + time.Sleep(time.Duration(tries*tries) * time.Second) + } + + // create URL + downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], downloadPath) + if err != nil { + return nil, fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], downloadPath, err) + } + + // start file download + resp, err := http.Get(downloadURL) + if err != nil { + return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err) + } + defer resp.Body.Close() + + // download and write file + buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength)) + n, err := io.Copy(buf, resp.Body) + if err != nil { + return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err) + } + if resp.ContentLength != n { + return nil, fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength) + } + + return buf.Bytes(), nil +} + +func joinURLandPath(baseURL, urlPath string) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + + u.Path = path.Join(u.Path, urlPath) + return u.String(), nil +} diff --git a/updates/file.go b/updates/file.go new file mode 100644 index 00000000..3f0d133a --- /dev/null +++ b/updates/file.go @@ -0,0 +1,46 @@ +package updates + +// File represents a file from the update system. +type File struct { + filepath string + version string + stable bool +} + +func newFile(filepath string, version string, stable bool) *File { + return &File{ + filepath: filepath, + version: version, + stable: stable, + } +} + +// Path returns the filepath of the file. +func (f *File) Path() string { + return f.filepath +} + +// Version returns the version of the file. +func (f *File) Version() string { + return f.version +} + +// Stable returns whether the file is from a stable release. +func (f *File) Stable() bool { + return f.stable +} + +// Open opens the file and returns the +func (f *File) Open() { + +} + +// ReportError reports an error back to Safing. This will not automatically blacklist the file. +func (f *File) ReportError() { + +} + +// Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on. +func (f *File) Blacklist() { + +} diff --git a/updates/filename.go b/updates/filename.go new file mode 100644 index 00000000..cca6cc4f --- /dev/null +++ b/updates/filename.go @@ -0,0 +1,41 @@ +package updates + +import ( + "fmt" + "regexp" + "strings" +) + +var versionRegex = regexp.MustCompile("_v[0-9]+-[0-9]+-[0-9]+b?") + +func getIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) { + // extract version + rawVersion := versionRegex.FindString(versionedPath) + if rawVersion == "" { + return "", "", false + } + + // replace - with . and trim _ + version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", -1) + + // put together without version + i := strings.Index(versionedPath, rawVersion) + if i < 0 { + // extracted version not in string (impossible) + return "", "", false + } + return versionedPath[:i] + versionedPath[i+len(rawVersion):], version, true +} + +func getVersionedPath(identifier, version string) (versionedPath string) { + // split in half + splittedFilePath := strings.SplitN(identifier, ".", 2) + // replace . with - + transformedVersion := strings.Replace(version, ".", "-", -1) + + // put together + if len(splittedFilePath) == 1 { + return fmt.Sprintf("%s_v%s", splittedFilePath[0], transformedVersion) + } + return fmt.Sprintf("%s_v%s.%s", splittedFilePath[0], transformedVersion, splittedFilePath[1]) +} diff --git a/updates/get.go b/updates/get.go new file mode 100644 index 00000000..d72ac0a3 --- /dev/null +++ b/updates/get.go @@ -0,0 +1,77 @@ +package updates + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/Safing/portbase/log" +) + +var ( + ErrNotFound = errors.New("the requested file could not be found") +) + +// GetPlatformFile returns the latest platform specific file identified by the given identifier. +func GetPlatformFile(identifier string) (*File, error) { + identifier = filepath.Join(fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), identifier) + // From https://golang.org/pkg/runtime/#GOARCH + // GOOS is the running program's operating system target: one of darwin, freebsd, linux, and so on. + // GOARCH is the running program's architecture target: one of 386, amd64, arm, s390x, and so on. + return loadOrFetchFile(identifier) +} + +// GetFile returns the latest generic file identified by the given identifier. +func GetFile(identifier string) (*File, error) { + identifier = filepath.Join("all", identifier) + return loadOrFetchFile(identifier) +} + +func getLatestFilePath(identifier string) (versionedFilePath, version string, stable bool, ok bool) { + updatesLock.RLock() + version, ok = stableUpdates[identifier] + if !ok { + version, ok = latestUpdates[identifier] + if !ok { + log.Tracef("updates: file %s does not exist", identifier) + return "", "", false, false + // TODO: if in development mode, reload latest index to check for newly sideloaded updates + // err := reloadLatest() + } + } + updatesLock.RUnlock() + + // TODO: Fix for stable release + return getVersionedPath(identifier, version), version, false, true +} + +func loadOrFetchFile(identifier string) (*File, error) { + versionedFilePath, version, stable, ok := getLatestFilePath(identifier) + if !ok { + // TODO: if in development mode, search updates dir for sideloaded apps + return nil, ErrNotFound + } + + // build final filepath + realFilePath := filepath.Join(updateStoragePath, versionedFilePath) + if _, err := os.Stat(realFilePath); err == nil { + // file exists + return newFile(realFilePath, version, stable), nil + } + + // download file + log.Tracef("updates: starting download of %s", versionedFilePath) + var err error + for tries := 0; tries < 5; tries++ { + err := fetchFile(realFilePath, versionedFilePath, tries) + if err != nil { + log.Tracef("updates: failed to download %s: %s, retrying (%d)", versionedFilePath, err, tries+1) + } else { + return newFile(realFilePath, version, stable), nil + } + } + log.Warningf("updates: failed to download %s: %s", versionedFilePath, err) + return nil, err +} diff --git a/updates/get_test.go b/updates/get_test.go new file mode 100644 index 00000000..92dbc324 --- /dev/null +++ b/updates/get_test.go @@ -0,0 +1,24 @@ +package updates + +import "testing" + +func testBuildVersionedFilePath(t *testing.T, identifier, version, expectedVersionedFilePath string) { + updatesLock.Lock() + stableUpdates[identifier] = version + // betaUpdates[identifier] = version + updatesLock.Unlock() + + versionedFilePath, _, _, ok := getLatestFilePath(identifier) + if !ok { + t.Errorf("identifier %s should exist", identifier) + } + if versionedFilePath != expectedVersionedFilePath { + t.Errorf("unexpected versionedFilePath: %s", versionedFilePath) + } +} + +func TestBuildVersionedFilePath(t *testing.T) { + testBuildVersionedFilePath(t, "path/to/asset.zip", "1.2.3", "path/to/asset_v1-2-3.zip") + testBuildVersionedFilePath(t, "path/to/asset.tar.gz", "1.2.3b", "path/to/asset_v1-2-3b.tar.gz") + testBuildVersionedFilePath(t, "path/to/asset", "1.2.3b", "path/to/asset_v1-2-3b") +} diff --git a/updates/latest.go b/updates/latest.go new file mode 100644 index 00000000..fc41b252 --- /dev/null +++ b/updates/latest.go @@ -0,0 +1,141 @@ +package updates + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + + "github.com/Safing/portbase/log" +) + +var ( + stableUpdates = make(map[string]string) + betaUpdates = make(map[string]string) + latestUpdates = make(map[string]string) + updatesLock sync.RWMutex +) + +// ReloadLatest reloads available updates from disk. +func ReloadLatest() error { + newLatestUpdates := make(map[string]string) + + // all + new, err1 := ScanForLatest(filepath.Join(updateStoragePath, "all"), false) + for key, val := range new { + newLatestUpdates[key] = val + } + + // os_platform + new, err2 := ScanForLatest(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)), false) + for key, val := range new { + newLatestUpdates[key] = val + } + + if err1 != nil && err2 != nil { + return fmt.Errorf("could not load latest update versions: %s, %s", err1, err2) + } + + log.Tracef("updates: loading latest updates:") + + for key, val := range newLatestUpdates { + log.Tracef("updates: %s v%s", key, val) + } + + updatesLock.Lock() + latestUpdates = newLatestUpdates + updatesLock.Unlock() + + log.Tracef("updates: load complete") + + if len(stableUpdates) == 0 { + err := loadIndexesFromDisk() + if err != nil { + return err + } + } + + return nil +} + +func ScanForLatest(baseDir string, hardFail bool) (latest map[string]string, lastError error) { + var added int + latest = make(map[string]string) + + filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + lastError = err + if hardFail { + return err + } + log.Warningf("updates: could not read %s", path) + return nil + } + if !info.IsDir() { + added++ + } + + relativePath := strings.TrimLeft(strings.TrimPrefix(path, baseDir), "/") + identifierPath, version, ok := getIdentifierAndVersion(relativePath) + if !ok { + return nil + } + + // add/update index + storedVersion, ok := latest[identifierPath] + if ok { + // FIXME: this will fail on multi-digit version segments! + if version > storedVersion { + latest[identifierPath] = version + } + } else { + latest[identifierPath] = version + } + + return nil + }) + + if lastError != nil { + if hardFail { + return nil, lastError + } + if added == 0 { + return latest, lastError + } + } + return latest, nil +} + +func loadIndexesFromDisk() error { + data, err := ioutil.ReadFile(filepath.Join(updateStoragePath, "stable.json")) + if err != nil { + if os.IsNotExist(err) { + log.Infof("updates: stable.json does not yet exist, waiting for first update cycle") + return nil + } + return err + } + + newStableUpdates := make(map[string]string) + err = json.Unmarshal(data, &newStableUpdates) + if err != nil { + return err + } + + if len(newStableUpdates) == 0 { + return errors.New("stable.json is empty") + } + + log.Tracef("updates: loaded stable.json") + + updatesLock.Lock() + stableUpdates = newStableUpdates + updatesLock.Unlock() + + return nil +} diff --git a/updates/latest_test.go b/updates/latest_test.go new file mode 100644 index 00000000..542bdb45 --- /dev/null +++ b/updates/latest_test.go @@ -0,0 +1,73 @@ +package updates + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, expectedVersion string) { + fullPath := filepath.Join(basePath, filePath) + + // create dir + dirPath := filepath.Dir(fullPath) + err := os.MkdirAll(dirPath, 0755) + if err != nil { + t.Fatalf("could not create test dir: %s\n", err) + return + } + + // touch file + err = ioutil.WriteFile(fullPath, []byte{}, 0644) + if err != nil { + t.Fatalf("could not create test file: %s\n", err) + return + } + + // run loadLatestScope + latest, err := ScanForLatest(basePath, true) + if err != nil { + t.Errorf("could not update latest: %s\n", err) + return + } + for key, val := range latest { + latestUpdates[key] = val + } + + // test result + version, ok := latestUpdates[expectedIdentifier] + if !ok { + t.Errorf("identifier %s not in map", expectedIdentifier) + t.Errorf("current map: %v", latestUpdates) + } + if version != expectedVersion { + t.Errorf("unexpected version for %s: %s", filePath, version) + } +} + +func TestLoadLatestScope(t *testing.T) { + + updatesLock.Lock() + defer updatesLock.Unlock() + + tmpDir, err := ioutil.TempDir("", "testing_") + if err != nil { + t.Fatalf("could not create test dir: %s\n", err) + return + } + defer os.RemoveAll(tmpDir) + + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "1.2.3") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3b.zip", "all/ui/assets.zip", "1.2.3b") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "1.2.4") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "1.3.4") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v2-3-4.zip", "all/ui/assets.zip", "2.3.4") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "2.3.4") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "2.3.4") + testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "2.3.4") + testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "1.2.3") + testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v2-1-1", "os_platform/portmaster/portmaster", "2.1.1") + testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "2.1.1") + +} diff --git a/updates/main.go b/updates/main.go new file mode 100644 index 00000000..4f6695b7 --- /dev/null +++ b/updates/main.go @@ -0,0 +1,95 @@ +package updates + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/modules" +) + +var ( + updateStoragePath string +) + +func init() { + modules.Register("updates", prep, start, nil, "database") +} + +func prep() error { + updateStoragePath = filepath.Join(database.GetDatabaseRoot(), "updates") + + err := checkUpdateDirs() + if err != nil { + return err + } + + return nil +} + +func start() error { + err := ReloadLatest() + if err != nil { + return err + } + + go updater() + return nil +} + +func stop() error { + return os.RemoveAll(filepath.Join(updateStoragePath, "tmp")) +} + +func checkUpdateDirs() error { + // all + err := checkDir(filepath.Join(updateStoragePath, "all")) + if err != nil { + return err + } + + // os_platform + err = checkDir(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))) + if err != nil { + return err + } + + // tmp + err = checkDir(filepath.Join(updateStoragePath, "tmp")) + if err != nil { + return err + } + + return nil +} + +func checkDir(dirPath string) error { + f, err := os.Stat(dirPath) + if err == nil { + // file exists + if f.IsDir() { + return nil + } + err = os.Remove(dirPath) + if err != nil { + return fmt.Errorf("could not remove file %s to place dir: %s", dirPath, err) + } + err = os.MkdirAll(dirPath, 0755) + if err != nil { + return fmt.Errorf("could not create dir %s: %s", dirPath, err) + } + return nil + } + // file does not exist + if os.IsNotExist(err) { + err = os.MkdirAll(dirPath, 0755) + if err != nil { + return fmt.Errorf("could not create dir %s: %s", dirPath, err) + } + return nil + } + // other error + return fmt.Errorf("failed to access %s: %s", dirPath, err) +} diff --git a/updates/updater.go b/updates/updater.go new file mode 100644 index 00000000..56a39738 --- /dev/null +++ b/updates/updater.go @@ -0,0 +1,88 @@ +package updates + +import ( + "encoding/json" + "errors" + "io/ioutil" + "path/filepath" + "time" + + "github.com/Safing/portbase/log" +) + +func updater() { + time.Sleep(10 * time.Second) + for { + err := checkForUpdates() + if err != nil { + log.Warningf("updates: failed to check for updates: %s", err) + } + time.Sleep(1 * time.Hour) + } +} + +func checkForUpdates() error { + + // download new index + var data []byte + var err error + for tries := 0; tries < 3; tries++ { + data, err = fetchData("stable.json", tries) + if err == nil { + break + } + } + if err != nil { + return err + } + + newStableUpdates := make(map[string]string) + err = json.Unmarshal(data, &newStableUpdates) + if err != nil { + return err + } + + if len(newStableUpdates) == 0 { + return errors.New("stable.json is empty") + } + + // FIXINSTABLE: correct log line + log.Infof("updates: downloaded new update index: stable.json (alpha until we actually reach stable)") + + // update existing files + log.Tracef("updates: updating existing files") + updatesLock.RLock() + for identifier, newVersion := range newStableUpdates { + oldVersion, ok := latestUpdates[identifier] + if ok && newVersion != oldVersion { + + filePath := getVersionedPath(identifier, newVersion) + realFilePath := filepath.Join(updateStoragePath, filePath) + for tries := 0; tries < 3; tries++ { + err := fetchFile(realFilePath, filePath, tries) + if err == nil { + break + } + } + if err != nil { + log.Warningf("failed to update %s to %s: %s", identifier, newVersion, err) + } + + } + } + updatesLock.RUnlock() + log.Tracef("updates: finished updating existing files") + + // update stable index + updatesLock.Lock() + stableUpdates = newStableUpdates + updatesLock.Unlock() + + // save stable index + err = ioutil.WriteFile(filepath.Join(updateStoragePath, "stable.json"), data, 0644) + if err != nil { + log.Warningf("updates: failed to save new version of stable.json: %s", err) + } + + return nil +} diff --git a/updates/uptool/.gitignore b/updates/uptool/.gitignore new file mode 100644 index 00000000..c5074cf6 --- /dev/null +++ b/updates/uptool/.gitignore @@ -0,0 +1 @@ +uptool diff --git a/updates/uptool/root.go b/updates/uptool/root.go new file mode 100644 index 00000000..8c75312c --- /dev/null +++ b/updates/uptool/root.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "uptool", + Short: "helper tool for the update process", + Run: func(cmd *cobra.Command, args []string) { + cmd.Usage() + }, +} + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/updates/uptool/scan.go b/updates/uptool/scan.go new file mode 100644 index 00000000..ccb5185d --- /dev/null +++ b/updates/uptool/scan.go @@ -0,0 +1,35 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/Safing/portmaster/updates" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(scanCmd) +} + +var scanCmd = &cobra.Command{ + Use: "scan", + Short: "Scan the current directory and print the result", + RunE: scan, +} + +func scan(cmd *cobra.Command, args []string) error { + + latest, err := updates.ScanForLatest(".", true) + if err != nil { + return err + } + + data, err := json.MarshalIndent(latest, "", " ") + if err != nil { + return err + } + + fmt.Println(string(data)) + return nil +}