Restructure modules (#1572)
* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
347
base/updater/fetch.go
Normal file
347
base/updater/fetch.go
Normal file
@@ -0,0 +1,347 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/safing/jess/filesig"
|
||||
"github.com/safing/jess/lhash"
|
||||
"github.com/safing/portmaster/base/log"
|
||||
"github.com/safing/portmaster/base/utils/renameio"
|
||||
)
|
||||
|
||||
func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, rv *ResourceVersion, tries int) error {
|
||||
// backoff when retrying
|
||||
if tries > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // module is shutting down
|
||||
case <-time.After(time.Duration(tries*tries) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// check destination dir
|
||||
dirPath := filepath.Dir(rv.storagePath())
|
||||
err := reg.storageDir.EnsureAbsPath(dirPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create updates folder: %s", dirPath)
|
||||
}
|
||||
|
||||
// If verification is enabled, download signature first.
|
||||
var (
|
||||
verifiedHash *lhash.LabeledHash
|
||||
sigFileData []byte
|
||||
)
|
||||
if rv.resource.VerificationOptions != nil {
|
||||
verifiedHash, sigFileData, err = reg.fetchAndVerifySigFile(
|
||||
ctx, client,
|
||||
rv.resource.VerificationOptions,
|
||||
rv.versionedSigPath(), rv.SigningMetadata(),
|
||||
tries,
|
||||
)
|
||||
if err != nil {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return fmt.Errorf("signature verification failed: %w", err)
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err)
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// open file for writing
|
||||
atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create temp file for download: %w", err)
|
||||
}
|
||||
defer atomicFile.Cleanup() //nolint:errcheck // ignore error for now, tmp dir will be cleaned later again anyway
|
||||
|
||||
// start file download
|
||||
resp, downloadURL, err := reg.makeRequest(ctx, client, rv.versionedPath(), tries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
// Write to the hasher at the same time, if needed.
|
||||
var hasher hash.Hash
|
||||
var writeDst io.Writer = atomicFile
|
||||
if verifiedHash != nil {
|
||||
hasher = verifiedHash.Algorithm().RawHasher()
|
||||
writeDst = io.MultiWriter(hasher, atomicFile)
|
||||
}
|
||||
|
||||
// Download and write file.
|
||||
n, err := io.Copy(writeDst, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download %q: %w", downloadURL, err)
|
||||
}
|
||||
if resp.ContentLength != n {
|
||||
return fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength)
|
||||
}
|
||||
|
||||
// Before file is finalized, check if hash, if available.
|
||||
if hasher != nil {
|
||||
downloadDigest := hasher.Sum(nil)
|
||||
if verifiedHash.EqualRaw(downloadDigest) {
|
||||
log.Infof("%s: verified signature of %s", reg.Name, downloadURL)
|
||||
} else {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return errors.New("file does not match signed checksum")
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: checksum does not match file from %s", reg.Name, downloadURL)
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL)
|
||||
}
|
||||
|
||||
// Reset hasher to signal that the sig should not be written.
|
||||
hasher = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Write signature file, if we have one and if verification succeeded.
|
||||
if len(sigFileData) > 0 && hasher != nil {
|
||||
sigFilePath := rv.storagePath() + filesig.Extension
|
||||
err := os.WriteFile(sigFilePath, sigFileData, 0o0644) //nolint:gosec
|
||||
if err != nil {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return fmt.Errorf("failed to write signature file %s: %w", sigFilePath, err)
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: failed to write signature file %s: %s", reg.Name, sigFilePath, err)
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: failed to write signature file %s: %s", reg.Name, sigFilePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// finalize file
|
||||
err = atomicFile.CloseAtomicallyReplace()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: failed to finalize file %s: %w", reg.Name, rv.storagePath(), err)
|
||||
}
|
||||
// set permissions
|
||||
if !onWindows {
|
||||
// TODO: only set executable files to 0755, set other to 0644
|
||||
err = os.Chmod(rv.storagePath(), 0o0755) //nolint:gosec // See TODO above.
|
||||
if err != nil {
|
||||
log.Warningf("%s: failed to set permissions on downloaded file %s: %s", reg.Name, rv.storagePath(), err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("%s: fetched %s and stored to %s", reg.Name, downloadURL, rv.storagePath())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (reg *ResourceRegistry) fetchMissingSig(ctx context.Context, client *http.Client, rv *ResourceVersion, tries int) error {
|
||||
// backoff when retrying
|
||||
if tries > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // module is shutting down
|
||||
case <-time.After(time.Duration(tries*tries) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// Check destination dir.
|
||||
dirPath := filepath.Dir(rv.storagePath())
|
||||
err := reg.storageDir.EnsureAbsPath(dirPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create updates folder: %s", dirPath)
|
||||
}
|
||||
|
||||
// Download and verify the missing signature.
|
||||
verifiedHash, sigFileData, err := reg.fetchAndVerifySigFile(
|
||||
ctx, client,
|
||||
rv.resource.VerificationOptions,
|
||||
rv.versionedSigPath(), rv.SigningMetadata(),
|
||||
tries,
|
||||
)
|
||||
if err != nil {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return fmt.Errorf("signature verification failed: %w", err)
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err)
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the signature matches the resource file.
|
||||
ok, err := verifiedHash.MatchesFile(rv.storagePath())
|
||||
if err != nil {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return fmt.Errorf("error while verifying resource file: %w", err)
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: error while verifying resource file %s", reg.Name, rv.storagePath())
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: error while verifying resource file %s", reg.Name, rv.storagePath())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !ok {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return errors.New("resource file does not match signed checksum")
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: checksum does not match resource file from %s", reg.Name, rv.storagePath())
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: checksum does not match resource file from %s", reg.Name, rv.storagePath())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write signature file.
|
||||
err = os.WriteFile(rv.storageSigPath(), sigFileData, 0o0644) //nolint:gosec
|
||||
if err != nil {
|
||||
switch rv.resource.VerificationOptions.DownloadPolicy {
|
||||
case SignaturePolicyRequire:
|
||||
return fmt.Errorf("failed to write signature file %s: %w", rv.storageSigPath(), err)
|
||||
case SignaturePolicyWarn:
|
||||
log.Warningf("%s: failed to write signature file %s: %s", reg.Name, rv.storageSigPath(), err)
|
||||
case SignaturePolicyDisable:
|
||||
log.Debugf("%s: failed to write signature file %s: %s", reg.Name, rv.storageSigPath(), err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("%s: fetched %s and stored to %s", reg.Name, rv.versionedSigPath(), rv.storageSigPath())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *http.Client, verifOpts *VerificationOptions, sigFilePath string, requiredMetadata map[string]string, tries int) (*lhash.LabeledHash, []byte, error) {
|
||||
// Download signature file.
|
||||
resp, _, err := reg.makeRequest(ctx, client, sigFilePath, tries)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
sigFileData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Extract all signatures.
|
||||
sigs, err := filesig.ParseSigFile(sigFileData)
|
||||
switch {
|
||||
case len(sigs) == 0 && err != nil:
|
||||
return nil, nil, fmt.Errorf("failed to parse signature file: %w", err)
|
||||
case len(sigs) == 0:
|
||||
return nil, nil, errors.New("no signatures found in signature file")
|
||||
case err != nil:
|
||||
return nil, nil, fmt.Errorf("failed to parse signature file: %w", err)
|
||||
}
|
||||
|
||||
// Verify all signatures.
|
||||
var verifiedHash *lhash.LabeledHash
|
||||
for _, sig := range sigs {
|
||||
fd, err := filesig.VerifyFileData(
|
||||
sig,
|
||||
requiredMetadata,
|
||||
verifOpts.TrustStore,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, sigFileData, err
|
||||
}
|
||||
|
||||
// Save or check verified hash.
|
||||
if verifiedHash == nil {
|
||||
verifiedHash = fd.FileHash()
|
||||
} else if !fd.FileHash().Equal(verifiedHash) {
|
||||
// Return an error if two valid hashes mismatch.
|
||||
// For simplicity, all hash algorithms must be the same for now.
|
||||
return nil, sigFileData, errors.New("file hashes from different signatures do not match")
|
||||
}
|
||||
}
|
||||
|
||||
return verifiedHash, sigFileData, nil
|
||||
}
|
||||
|
||||
func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, downloadPath string, tries int) (fileData []byte, downloadedFrom string, err error) {
|
||||
// backoff when retrying
|
||||
if tries > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, "", nil // module is shutting down
|
||||
case <-time.After(time.Duration(tries*tries) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// start file download
|
||||
resp, downloadURL, err := reg.makeRequest(ctx, client, downloadPath, tries)
|
||||
if err != nil {
|
||||
return nil, downloadURL, err
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
// download and write file
|
||||
buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
|
||||
n, err := io.Copy(buf, resp.Body)
|
||||
if err != nil {
|
||||
return nil, downloadURL, fmt.Errorf("failed to download %q: %w", downloadURL, err)
|
||||
}
|
||||
if resp.ContentLength != n {
|
||||
return nil, downloadURL, fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength)
|
||||
}
|
||||
|
||||
return buf.Bytes(), downloadURL, nil
|
||||
}
|
||||
|
||||
func (reg *ResourceRegistry) makeRequest(ctx context.Context, client *http.Client, downloadPath string, tries int) (resp *http.Response, downloadURL string, err error) {
|
||||
// parse update URL
|
||||
updateBaseURL := reg.UpdateURLs[tries%len(reg.UpdateURLs)]
|
||||
u, err := url.Parse(updateBaseURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to parse update URL %q: %w", updateBaseURL, err)
|
||||
}
|
||||
// add download path
|
||||
u.Path = path.Join(u.Path, downloadPath)
|
||||
// compile URL
|
||||
downloadURL = u.String()
|
||||
|
||||
// create request
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, http.NoBody)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to create request for %q: %w", downloadURL, err)
|
||||
}
|
||||
|
||||
// set user agent
|
||||
if reg.UserAgent != "" {
|
||||
req.Header.Set("User-Agent", reg.UserAgent)
|
||||
}
|
||||
|
||||
// start request
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to make request to %q: %w", downloadURL, err)
|
||||
}
|
||||
|
||||
// check return code
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_ = resp.Body.Close()
|
||||
return nil, "", fmt.Errorf("failed to fetch %q: %d %s", downloadURL, resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
return resp, downloadURL, err
|
||||
}
|
||||
Reference in New Issue
Block a user