From 602db080c5448a3da36edb67fbf5651264b8071f Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 15 Nov 2023 15:12:00 +0100 Subject: [PATCH 1/5] Add export and import for profiles --- profile/api.go | 91 ++++++++ profile/database.go | 6 +- profile/fingerprint.go | 10 +- profile/fingerprint_test.go | 2 +- profile/get.go | 14 +- profile/icon.go | 7 +- profile/icons.go | 69 ++++++ profile/migrations.go | 2 +- profile/module.go | 9 + profile/profile.go | 16 +- sync/module.go | 3 + sync/profile.go | 435 ++++++++++++++++++++++++++++++++++-- sync/settings.go | 89 ++++---- 13 files changed, 668 insertions(+), 85 deletions(-) create mode 100644 profile/icons.go diff --git a/profile/api.go b/profile/api.go index e8213bd6..d563b5ae 100644 --- a/profile/api.go +++ b/profile/api.go @@ -1,10 +1,14 @@ package profile import ( + "errors" "fmt" + "net/http" + "strings" "github.com/safing/portbase/api" "github.com/safing/portbase/formats/dsd" + "github.com/safing/portbase/utils" ) func registerAPIEndpoints() error { @@ -19,6 +23,28 @@ func registerAPIEndpoints() error { return err } + if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Get Profile Icon", + Description: "Returns the requested profile icon.", + Path: "profile/icon/{id:[0-9a-f]{40-80}}.{ext:[a-z]{3-4}}", + Read: api.PermitUser, + BelongsTo: module, + DataFunc: handleGetProfileIcon, + }); err != nil { + return err + } + + if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Update Profile Icon", + Description: "Merge multiple profiles into a new one.", + Path: "profile/icon/update", + Write: api.PermitUser, + BelongsTo: module, + StructFunc: handleUpdateProfileIcon, + }); err != nil { + return err + } + return nil } @@ -64,3 +90,68 @@ func handleMergeProfiles(ar *api.Request) (i interface{}, err error) { New: newProfile.ScopedID(), }, nil } + +func handleGetProfileIcon(ar *api.Request) (data []byte, err error) { + // Get profile icon. + data, err = GetProfileIcon(ar.URLVars["id"], ar.URLVars["ext"]) + if err != nil { + return nil, err + } + + // Set content type for icon. + contentType, ok := utils.MimeTypeByExtension(ar.URLVars["ext"]) + if ok { + ar.ResponseHeader.Set("Content-Type", contentType) + } + + return data, nil +} + +type updateProfileIconResponse struct { + Filename string `json:"filename"` +} + +func handleUpdateProfileIcon(ar *api.Request) (any, error) { + // Check input. + if len(ar.InputData) == 0 { + return nil, api.ErrorWithStatus(errors.New("no content"), http.StatusBadRequest) + } + mimeType := ar.Header.Get("Content-Type") + if mimeType == "" { + return nil, api.ErrorWithStatus(errors.New("no content type"), http.StatusBadRequest) + } + + // Derive image format from content type. + mimeType = strings.TrimSpace(mimeType) + mimeType = strings.ToLower(mimeType) + mimeType, _, _ = strings.Cut(mimeType, ";") + var ext string + switch mimeType { + case "image/gif": + ext = "gif" + case "image/jpeg": + ext = "jpg" + case "image/jpg": + ext = "jpg" + case "image/png": + ext = "png" + case "image/svg+xml": + ext = "svg" + case "image/tiff": + ext = "tiff" + case "image/webp": + ext = "webp" + default: + return "", api.ErrorWithStatus(errors.New("unsupported image format"), http.StatusBadRequest) + } + + // Update profile icon. + filename, err := UpdateProfileIcon(ar.InputData, ext) + if err != nil { + return nil, err + } + + return &updateProfileIconResponse{ + Filename: filename, + }, nil +} diff --git a/profile/database.go b/profile/database.go index d311bd32..a9f927a1 100644 --- a/profile/database.go +++ b/profile/database.go @@ -24,11 +24,13 @@ var profileDB = database.NewInterface(&database.Options{ Internal: true, }) -func makeScopedID(source profileSource, id string) string { +// MakeScopedID returns a scoped profile ID. +func MakeScopedID(source ProfileSource, id string) string { return string(source) + "/" + id } -func makeProfileKey(source profileSource, id string) string { +// MakeProfileKey returns a profile key. +func MakeProfileKey(source ProfileSource, id string) string { return ProfilesDBPath + string(source) + "/" + id } diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 3f62ba9d..9caf8794 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -67,7 +67,7 @@ type ( // merged from. The merged profile should create a new profile ID derived // from the new fingerprints and add all fingerprints with this field set // to the originating profile ID - MergedFrom string + MergedFrom string // `json:"mergedFrom,omitempty"` } // Tag represents a simple key/value kind of tag used in process metadata @@ -170,7 +170,8 @@ type parsedFingerprints struct { cmdlinePrints []matchingFingerprint } -func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { +// ParseFingerprints parses the fingerprints to make them ready for matching. +func ParseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { parsed = &parsedFingerprints{} // Add deprecated LinkedPath to fingerprints, if they are empty. @@ -230,7 +231,7 @@ func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed * default: if firstErr == nil { - firstErr = fmt.Errorf("unknown fingerprint operation: %q", entry.Type) + firstErr = fmt.Errorf("unknown fingerprint operation: %q", entry.Operation) } } } @@ -367,7 +368,8 @@ const ( deriveFPKeyIDForValue ) -func deriveProfileID(fps []Fingerprint) string { +// DeriveProfileID derives a profile ID from the given fingerprints. +func DeriveProfileID(fps []Fingerprint) string { // Sort the fingerprints. sortAndCompactFingerprints(fps) diff --git a/profile/fingerprint_test.go b/profile/fingerprint_test.go index 4857d8be..3cbdb551 100644 --- a/profile/fingerprint_test.go +++ b/profile/fingerprint_test.go @@ -47,7 +47,7 @@ func TestDeriveProfileID(t *testing.T) { }) // Check if fingerprint matches. - id := deriveProfileID(fps) + id := DeriveProfileID(fps) assert.Equal(t, "PTSRP7rdCnmvdjRoPMTrtjj7qk7PxR1a9YdBWUGwnZXJh2", id) } } diff --git a/profile/get.go b/profile/get.go index dc37efea..2a5424cf 100644 --- a/profile/get.go +++ b/profile/get.go @@ -35,7 +35,7 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P // Get active profile based on the ID, if available. if id != "" { // Check if there already is an active profile. - profile = getActiveProfile(makeScopedID(SourceLocal, id)) + profile = getActiveProfile(MakeScopedID(SourceLocal, id)) if profile != nil { // Mark active and return if not outdated. if profile.outdated.IsNotSet() { @@ -57,9 +57,9 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P return nil, errors.New("cannot get local profiles without ID and matching data") } - profile, err = getProfile(makeScopedID(SourceLocal, id)) + profile, err = getProfile(MakeScopedID(SourceLocal, id)) if err != nil { - return nil, fmt.Errorf("failed to load profile %s by ID: %w", makeScopedID(SourceLocal, id), err) + return nil, fmt.Errorf("failed to load profile %s by ID: %w", MakeScopedID(SourceLocal, id), err) } } @@ -70,7 +70,7 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P // Get special profile from DB. if profile == nil { - profile, err = getProfile(makeScopedID(SourceLocal, id)) + profile, err = getProfile(MakeScopedID(SourceLocal, id)) if err != nil && !errors.Is(err, database.ErrNotFound) { log.Warningf("profile: failed to get special profile %s: %s", id, err) } @@ -188,12 +188,12 @@ func getProfile(scopedID string) (profile *Profile, err error) { // findProfile searches for a profile with the given linked path. If it cannot // find one, it will create a new profile for the given linked path. -func findProfile(source profileSource, md MatchingData) (profile *Profile, err error) { +func findProfile(source ProfileSource, md MatchingData) (profile *Profile, err error) { // TODO: Loading every profile from database and parsing it for every new // process might be quite expensive. Measure impact and possibly improve. // Get iterator over all profiles. - it, err := profileDB.Query(query.New(ProfilesDBPath + makeScopedID(source, ""))) + it, err := profileDB.Query(query.New(ProfilesDBPath + MakeScopedID(source, ""))) if err != nil { return nil, fmt.Errorf("failed to query for profiles: %w", err) } @@ -265,7 +265,7 @@ func loadProfileFingerprints(r record.Record) (parsed *parsedFingerprints, err e } // Parse and return fingerprints. - return parseFingerprints(profile.Fingerprints, profile.LinkedPath) + return ParseFingerprints(profile.Fingerprints, profile.LinkedPath) } func loadProfile(r record.Record) (*Profile, error) { diff --git a/profile/icon.go b/profile/icon.go index 0f084be5..91a0ae58 100644 --- a/profile/icon.go +++ b/profile/icon.go @@ -19,14 +19,17 @@ type IconType string const ( IconTypeFile IconType = "path" IconTypeDatabase IconType = "database" + IconTypeAPI IconType = "api" ) func (t IconType) sortOrder() int { switch t { - case IconTypeDatabase: + case IconTypeAPI: return 1 - case IconTypeFile: + case IconTypeDatabase: return 2 + case IconTypeFile: + return 3 default: return 100 } diff --git a/profile/icons.go b/profile/icons.go new file mode 100644 index 00000000..6260d217 --- /dev/null +++ b/profile/icons.go @@ -0,0 +1,69 @@ +package profile + +import ( + "crypto" + "encoding/hex" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/safing/portbase/api" +) + +var profileIconStoragePath = "" + +// GetProfileIcon returns the profile icon with the given ID and extension. +func GetProfileIcon(id, ext string) (data []byte, err error) { + // Build storage path. + iconPath := filepath.Join(profileIconStoragePath, id+"."+ext) + iconPath, err = filepath.Abs(iconPath) + if err != nil { + return nil, fmt.Errorf("failed to check icon path: %w", err) + } + // Do a quick check if we are still within the right directory. + // This check is not entirely correct, but is sufficient for this use case. + if !strings.HasPrefix(iconPath, profileIconStoragePath) { + return nil, api.ErrorWithStatus(errors.New("invalid icon"), http.StatusBadRequest) + } + + return os.ReadFile(iconPath) +} + +// UpdateProfileIcon creates or updates the given icon. +func UpdateProfileIcon(data []byte, ext string) (filename string, err error) { + // Check icon size. + if len(data) > 1_000_000 { + return "", errors.New("icon too big") + } + + // Calculate sha1 sum of icon. + h := crypto.SHA1.New() + if _, err := h.Write(data); err != nil { + return "", err + } + sum := hex.EncodeToString(h.Sum(nil)) + + // Check ext. + ext = strings.ToLower(ext) + switch ext { + case "gif": + case "jpeg": + ext = "jpg" + case "jpg": + case "png": + case "svg": + case "tiff": + case "webp": + default: + return "", errors.New("unsupported icon format") + } + + // Save to disk. + filename = sum + "." + ext + return filename, os.WriteFile(filepath.Join(profileIconStoragePath, filename), data, 0o0644) //nolint:gosec +} + +// TODO: Clean up icons regularly. diff --git a/profile/migrations.go b/profile/migrations.go index f2b33158..7d1f2979 100644 --- a/profile/migrations.go +++ b/profile/migrations.go @@ -208,7 +208,7 @@ func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *databa // Generate new ID. oldScopedID := profile.ScopedID() - newID := deriveProfileID(profile.Fingerprints) + newID := DeriveProfileID(profile.Fingerprints) // If they match, skip migration for this profile. if profile.ID == newID { diff --git a/profile/module.go b/profile/module.go index a462cd7e..6036d79a 100644 --- a/profile/module.go +++ b/profile/module.go @@ -2,10 +2,12 @@ package profile import ( "errors" + "fmt" "os" "github.com/safing/portbase/database" "github.com/safing/portbase/database/migration" + "github.com/safing/portbase/dataroot" "github.com/safing/portbase/log" "github.com/safing/portbase/modules" _ "github.com/safing/portmaster/core/base" @@ -45,6 +47,13 @@ func prep() error { return err } + // Setup icon storage location. + iconsDir := dataroot.Root().ChildDir("databases", 0o0700).ChildDir("icons", 0o0700) + if err := iconsDir.Ensure(); err != nil { + return fmt.Errorf("failed to create/check icons directory: %w", err) + } + profileIconStoragePath = iconsDir.Path + return nil } diff --git a/profile/profile.go b/profile/profile.go index db309080..87228acb 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -20,13 +20,13 @@ import ( "github.com/safing/portmaster/profile/endpoints" ) -// profileSource is the source of the profile. -type profileSource string +// ProfileSource is the source of the profile. +type ProfileSource string // Profile Sources. const ( - SourceLocal profileSource = "local" // local, editable - SourceSpecial profileSource = "special" // specials (read-only) + SourceLocal ProfileSource = "local" // local, editable + SourceSpecial ProfileSource = "special" // specials (read-only) ) // Default Action IDs. @@ -45,7 +45,7 @@ type Profile struct { //nolint:maligned // not worth the effort // ID is a unique identifier for the profile. ID string // constant // Source describes the source of the profile. - Source profileSource // constant + Source ProfileSource // constant // Name is a human readable name of the profile. It // defaults to the basename of the application. Name string @@ -262,7 +262,7 @@ func New(profile *Profile) *Profile { if profile.ID == "" { if len(profile.Fingerprints) > 0 { // Derive from fingerprints. - profile.ID = deriveProfileID(profile.Fingerprints) + profile.ID = DeriveProfileID(profile.Fingerprints) } else { // Generate random ID as fallback. log.Warningf("profile: creating new profile without fingerprints to derive ID from") @@ -284,12 +284,12 @@ func New(profile *Profile) *Profile { // ScopedID returns the scoped ID (Source + ID) of the profile. func (profile *Profile) ScopedID() string { - return makeScopedID(profile.Source, profile.ID) + return MakeScopedID(profile.Source, profile.ID) } // makeKey derives and sets the record Key from the profile attributes. func (profile *Profile) makeKey() { - profile.SetKey(makeProfileKey(profile.Source, profile.ID)) + profile.SetKey(MakeProfileKey(profile.Source, profile.ID)) } // Save saves the profile to the database. diff --git a/sync/module.go b/sync/module.go index bdecf442..0c6ebc63 100644 --- a/sync/module.go +++ b/sync/module.go @@ -25,5 +25,8 @@ func prep() error { if err := registerSingleSettingAPI(); err != nil { return err } + if err := registerProfileAPI(); err != nil { + return err + } return nil } diff --git a/sync/profile.go b/sync/profile.go index 5e7b66c5..34e8b625 100644 --- a/sync/profile.go +++ b/sync/profile.go @@ -1,45 +1,440 @@ package sync import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" "time" + "github.com/safing/portbase/api" + "github.com/safing/portbase/config" + "github.com/safing/portbase/log" "github.com/safing/portmaster/profile" ) // ProfileExport holds an export of a profile. type ProfileExport struct { //nolint:maligned - Type Type + Type Type `json:"type"` - // Identification (sync or import as new only) - ID string - Source string + // Identification + ID string `json:"id,omitempty"` + Source profile.ProfileSource `json:"source,omitempty"` // Human Metadata - Name string - Description string - Homepage string - Icons []profile.Icon - PresentationPath string - UsePresentationPath bool + Name string `json:"name"` + Description string `json:"description,omitempty"` + Homepage string `json:"homepage,omitempty"` + Icons []ProfileIcon `json:"icons,omitempty"` + PresentationPath string `json:"presPath,omitempty"` + UsePresentationPath bool `json:"usePresPath,omitempty"` // Process matching - Fingerprints []profile.Fingerprint + Fingerprints []ProfileFingerprint `json:"fingerprints"` // Settings - Config map[string]any + Config map[string]any `json:"config,omitempty"` - // Metadata (sync only) - LastEdited time.Time - Created time.Time - Internal bool + // Metadata + LastEdited *time.Time `json:"lastEdited,omitempty"` + Created *time.Time `json:"created,omitempty"` + Internal bool `json:"internal,omitempty"` +} + +// ProfileIcon represents a profile icon. +type ProfileIcon struct { + Type profile.IconType `json:"type"` + Value string `json:"value"` +} + +// ProfileIcon represents a profile fingerprint. +type ProfileFingerprint struct { + Type string `json:"type"` + Key string `json:"key,omitempty"` + Operation string `json:"operation"` + Value string `json:"value"` + MergedFrom string `json:"mergedFrom,omitempty"` +} + +// ProfileExportRequest is a request for a profile export. +type ProfileExportRequest struct { + ID string `json:"id"` } // ProfileImportRequest is a request to import Profile. type ProfileImportRequest struct { - ImportRequest + ImportRequest `json:",inline"` - // Reset all settings and fingerprints of target before import. - Reset bool + // AllowUnknown allows the import of unknown settings. + // Otherwise, attempting to import an unknown setting will result in an error. + AllowUnknown bool `json:"allowUnknown"` - Export *ProfileExport + // AllowReplace allows the import to replace other existing profiles. + AllowReplace bool `json:"allowReplaceProfiles"` + + Export *ProfileExport `json:"export"` +} + +// ProfileImportResult is returned by successful import operations. +type ProfileImportResult struct { + ImportResult `json:",inline"` + + ReplacesProfiles []string `json:"replacesProfiles"` +} + +func registerProfileAPI() error { + if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Export App Profile", + Description: "Exports app fingerprints, settings and metadata in a share-able format.", + Path: "sync/profile/export", + Read: api.PermitAdmin, + Write: api.PermitAdmin, + Parameters: []api.Parameter{{ + Method: http.MethodGet, + Field: "id", + Description: "Specify scoped profile ID to export.", + }}, + BelongsTo: module, + DataFunc: handleExportProfile, + }); err != nil { + return err + } + + if err := api.RegisterEndpoint(api.Endpoint{ + Name: "Import App Profile", + Description: "Imports full app profiles, including fingerprints, setting and metadata from the share-able format.", + Path: "sync/profile/import", + Read: api.PermitAdmin, + Write: api.PermitAdmin, + Parameters: []api.Parameter{ + { + Method: http.MethodPost, + Field: "allowReplace", + Description: "Allow replacing existing profiles.", + }, { + Method: http.MethodPost, + Field: "validate", + Description: "Validate only.", + }, { + Method: http.MethodPost, + Field: "reset", + Description: "Replace all existing settings.", + }, { + Method: http.MethodPost, + Field: "allowUnknown", + Description: "Allow importing of unknown values.", + }}, + BelongsTo: module, + StructFunc: handleImportProfile, + }); err != nil { + return err + } + + return nil +} + +func handleExportProfile(ar *api.Request) (data []byte, err error) { + var request *ProfileExportRequest + + // Get parameters. + q := ar.URL.Query() + if len(q) > 0 { + request = &ProfileExportRequest{ + ID: q.Get("id"), + } + } else { + request = &ProfileExportRequest{} + if err := json.Unmarshal(ar.InputData, request); err != nil { + return nil, fmt.Errorf("%w: failed to parse export request: %w", ErrExportFailed, err) + } + } + + // Check parameters. + if request.ID == "" { + return nil, errors.New("missing parameters") + } + + // Export. + export, err := ExportProfile(request.ID) + if err != nil { + return nil, err + } + + return serializeExport(export, ar) +} + +func handleImportProfile(ar *api.Request) (any, error) { + var request *ProfileImportRequest + + // Get parameters. + q := ar.URL.Query() + if len(q) > 0 { + request = &ProfileImportRequest{ + ImportRequest: ImportRequest{ + ValidateOnly: q.Has("validate"), + RawExport: string(ar.InputData), + RawMime: ar.Header.Get("Content-Type"), + }, + AllowUnknown: q.Has("allowUnknown"), + AllowReplace: q.Has("allowReplace"), + } + } else { + request = &ProfileImportRequest{} + if err := json.Unmarshal(ar.InputData, request); err != nil { + return nil, fmt.Errorf("%w: failed to parse import request: %w", ErrInvalidImportRequest, err) + } + } + + // Check if we need to parse the export. + switch { + case request.Export != nil && request.RawExport != "": + return nil, fmt.Errorf("%w: both Export and RawExport are defined", ErrInvalidImportRequest) + case request.RawExport != "": + // Parse export. + export := &ProfileExport{} + if err := parseExport(&request.ImportRequest, export); err != nil { + return nil, err + } + request.Export = export + case request.Export != nil: + // Export is aleady parsed. + default: + return nil, ErrInvalidImportRequest + } + + // Import. + return ImportProfile(request, profile.SourceLocal) +} + +// ExportProfile exports a profile. +func ExportProfile(scopedID string) (*ProfileExport, error) { + // Get Profile. + r, err := db.Get(profile.ProfilesDBPath + scopedID) + if err != nil { + return nil, fmt.Errorf("%w: failed to find profile: %w", ErrTargetNotFound, err) + } + p, err := profile.EnsureProfile(r) + if err != nil { + return nil, fmt.Errorf("%w: failed to load profile: %w", ErrExportFailed, err) + } + + // Copy exportable profile data. + export := &ProfileExport{ + Type: TypeProfile, + + // Identification + ID: p.ID, + Source: p.Source, + + // Human Metadata + Name: p.Name, + Description: p.Description, + Homepage: p.Homepage, + Icons: convertIconsToExport(p.Icons), + PresentationPath: p.PresentationPath, + UsePresentationPath: p.UsePresentationPath, + + // Process matching + Fingerprints: convertFingerprintsToExport(p.Fingerprints), + + // Settings + Config: p.Config, + + // Metadata + Internal: p.Internal, + } + // Add optional timestamps. + if p.LastEdited > 0 { + lastEdited := time.Unix(p.LastEdited, 0) + export.LastEdited = &lastEdited + } + if p.Created > 0 { + created := time.Unix(p.Created, 0) + export.Created = &created + } + + return export, nil +} + +// ImportProfile imports a profile. +func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.ProfileSource) (*ProfileImportResult, error) { + // Check import. + if r.Export.Type != TypeProfile { + return nil, ErrMismatch + } + + // Check Source. + if r.Export.Source != "" && r.Export.Source != requiredProfileSource { + return nil, ErrMismatch + } + // Check ID. + fingerprints := convertFingerprintsToInternal(r.Export.Fingerprints) + profileID := profile.DeriveProfileID(fingerprints) + if r.Export.ID != "" && r.Export.ID != profileID { + return nil, ErrMismatch + } else { + r.Export.ID = profileID + } + // Check Fingerprints. + _, err := profile.ParseFingerprints(fingerprints, "") + if err != nil { + return nil, fmt.Errorf("%w: the export contains invalid fingerprints: %w", ErrInvalidProfileData, err) + } + + // Flatten config. + settings := config.Flatten(r.Export.Config) + + // Check settings. + settingsResult, globalOnlySettingFound, err := checkSettings(settings) + if err != nil { + return nil, err + } + if settingsResult.ContainsUnknown && !r.AllowUnknown && !r.ValidateOnly { + return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImportRequest) + } + // Check if a setting is settable per app. + if globalOnlySettingFound { + return nil, fmt.Errorf("%w: export contains settings that cannot be set per app", ErrNotSettablePerApp) + } + + // Create result based on settings result. + result := &ProfileImportResult{ + ImportResult: *settingsResult, + } + + // Check if the profile already exists. + exists, err := db.Exists(profile.MakeProfileKey(r.Export.Source, r.Export.ID)) + if err != nil { + return nil, fmt.Errorf("internal import error: %w", err) + } + if exists { + result.ReplacesExisting = true + } + + // Check if import will delete any profiles. + requiredSourcePrefix := string(r.Export.Source) + "/" + result.ReplacesProfiles = make([]string, 0, len(r.Export.Fingerprints)) + for _, fp := range r.Export.Fingerprints { + if fp.MergedFrom != "" { + if !strings.HasPrefix(fp.MergedFrom, requiredSourcePrefix) { + return nil, fmt.Errorf("%w: exported profile was merged from different profile source", ErrInvalidImportRequest) + } + exists, err := db.Exists(profile.ProfilesDBPath + fp.MergedFrom) + if err != nil { + return nil, fmt.Errorf("internal import error: %w", err) + } + if exists { + result.ReplacesProfiles = append(result.ReplacesProfiles, fp.MergedFrom) + } + } + } + + // Stop here if we are only validating. + if r.ValidateOnly { + return result, nil + } + if result.ReplacesExisting && !r.AllowReplace { + return nil, fmt.Errorf("%w: import would replace existing profile", ErrImportFailed) + } + + // Create profile from export. + // Note: Don't use profile.New(), as this will not trigger a profile refresh if active. + in := r.Export + p := &profile.Profile{ + // Identification + ID: in.ID, + Source: requiredProfileSource, + + // Human Metadata + Name: in.Name, + Description: in.Description, + Homepage: in.Homepage, + Icons: convertIconsToInternal(in.Icons), + PresentationPath: in.PresentationPath, + UsePresentationPath: in.UsePresentationPath, + + // Process matching + Fingerprints: fingerprints, + + // Settings + Config: in.Config, + + // Metadata + Internal: in.Internal, + } + // Add optional timestamps. + if in.LastEdited != nil { + p.LastEdited = in.LastEdited.Unix() + } + if in.Created != nil { + p.Created = in.Created.Unix() + } + + // Save profile to db. + p.SetKey(profile.MakeProfileKey(p.Source, p.ID)) + err = p.Save() + if err != nil { + return nil, fmt.Errorf("%w: failed to save profile: %w", ErrImportFailed, err) + } + + // Delete profiles that were merged into the imported profile. + for _, profileID := range result.ReplacesProfiles { + err := db.Delete(profile.ProfilesDBPath + profileID) + if err != nil { + log.Errorf("sync: failed to delete merged profile %s on import: %s", profileID, err) + } + } + + return result, nil +} + +func convertIconsToExport(icons []profile.Icon) []ProfileIcon { + converted := make([]ProfileIcon, 0, len(icons)) + for _, icon := range icons { + converted = append(converted, ProfileIcon{ + Type: icon.Type, + Value: icon.Value, + }) + } + return converted +} + +func convertIconsToInternal(icons []ProfileIcon) []profile.Icon { + converted := make([]profile.Icon, 0, len(icons)) + for _, icon := range icons { + converted = append(converted, profile.Icon{ + Type: icon.Type, + Value: icon.Value, + }) + } + return converted +} + +func convertFingerprintsToExport(fingerprints []profile.Fingerprint) []ProfileFingerprint { + converted := make([]ProfileFingerprint, 0, len(fingerprints)) + for _, fp := range fingerprints { + converted = append(converted, ProfileFingerprint{ + Type: fp.Type, + Key: fp.Key, + Operation: fp.Operation, + Value: fp.Value, + MergedFrom: fp.MergedFrom, + }) + } + return converted +} + +func convertFingerprintsToInternal(fingerprints []ProfileFingerprint) []profile.Fingerprint { + converted := make([]profile.Fingerprint, 0, len(fingerprints)) + for _, fp := range fingerprints { + converted = append(converted, profile.Fingerprint{ + Type: fp.Type, + Key: fp.Key, + Operation: fp.Operation, + Value: fp.Value, + MergedFrom: fp.MergedFrom, + }) + } + return converted } diff --git a/sync/settings.go b/sync/settings.go index c9ada9a8..ae033fa5 100644 --- a/sync/settings.go +++ b/sync/settings.go @@ -221,52 +221,16 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) { if r.Export.Type != TypeSettings { return nil, ErrMismatch } - // Flatten config. settings := config.Flatten(r.Export.Config) - // Validate config and gather some metadata. - var ( - result = &ImportResult{} - checked int - globalOnlySettingFound bool - ) - err := config.ForEachOption(func(option *config.Option) error { - // Check if any setting is set. - if r.Reset && option.IsSetByUser() { - result.ReplacesExisting = true - } - - newValue, ok := settings[option.Key] - if ok { - checked++ - - // Validate the new value. - if err := option.ValidateValue(newValue); err != nil { - return fmt.Errorf("%w: configuration value for %s is invalid: %w", ErrInvalidSettingValue, option.Key, err) - } - - // Collect metadata. - if option.RequiresRestart { - result.RestartRequired = true - } - if !r.Reset && option.IsSetByUser() { - result.ReplacesExisting = true - } - if !option.AnnotationEquals(config.SettablePerAppAnnotation, true) { - globalOnlySettingFound = true - } - } - return nil - }) + // Check settings. + result, globalOnlySettingFound, err := checkSettings(settings) if err != nil { return nil, err } - if checked < len(settings) { - result.ContainsUnknown = true - if !r.AllowUnknown && !r.ValidateOnly { - return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImportRequest) - } + if result.ContainsUnknown && !r.AllowUnknown && !r.ValidateOnly { + return nil, fmt.Errorf("%w: the export contains unknown settings", ErrInvalidImportRequest) } // Import global settings. @@ -334,3 +298,48 @@ func ImportSettings(r *SettingsImportRequest) (*ImportResult, error) { return result, nil } + +func checkSettings(settings map[string]any) (result *ImportResult, globalOnlySettingFound bool, err error) { + result = &ImportResult{} + + // Validate config and gather some metadata. + var checked int + err = config.ForEachOption(func(option *config.Option) error { + // Check if any setting is set. + // TODO: Fix this - it only checks for global settings. + // if r.Reset && option.IsSetByUser() { + // result.ReplacesExisting = true + // } + + newValue, ok := settings[option.Key] + if ok { + checked++ + + // Validate the new value. + if err := option.ValidateValue(newValue); err != nil { + return fmt.Errorf("%w: configuration value for %s is invalid: %w", ErrInvalidSettingValue, option.Key, err) + } + + // Collect metadata. + if option.RequiresRestart { + result.RestartRequired = true + } + // TODO: Fix this - it only checks for global settings. + // if !r.Reset && option.IsSetByUser() { + // result.ReplacesExisting = true + // } + if !option.AnnotationEquals(config.SettablePerAppAnnotation, true) { + globalOnlySettingFound = true + } + } + return nil + }) + if err != nil { + return nil, false, err + } + if checked < len(settings) { + result.ContainsUnknown = true + } + + return result, globalOnlySettingFound, nil +} From 7751f57874727eacc1e2834427635eefe78c6cb1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 16 Nov 2023 13:54:04 +0100 Subject: [PATCH 2/5] Implement review suggestions --- profile/api.go | 2 +- sync/profile.go | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/profile/api.go b/profile/api.go index d563b5ae..6b0a6d94 100644 --- a/profile/api.go +++ b/profile/api.go @@ -36,7 +36,7 @@ func registerAPIEndpoints() error { if err := api.RegisterEndpoint(api.Endpoint{ Name: "Update Profile Icon", - Description: "Merge multiple profiles into a new one.", + Description: "Updates a profile icon.", Path: "profile/icon/update", Write: api.PermitUser, BelongsTo: module, diff --git a/sync/profile.go b/sync/profile.go index 34e8b625..984e3cff 100644 --- a/sync/profile.go +++ b/sync/profile.go @@ -48,7 +48,7 @@ type ProfileIcon struct { Value string `json:"value"` } -// ProfileIcon represents a profile fingerprint. +// ProfileFingerprint represents a profile fingerprint. type ProfileFingerprint struct { Type string `json:"type"` Key string `json:"key,omitempty"` @@ -165,12 +165,12 @@ func handleExportProfile(ar *api.Request) (data []byte, err error) { } func handleImportProfile(ar *api.Request) (any, error) { - var request *ProfileImportRequest + var request ProfileImportRequest // Get parameters. q := ar.URL.Query() if len(q) > 0 { - request = &ProfileImportRequest{ + request = ProfileImportRequest{ ImportRequest: ImportRequest{ ValidateOnly: q.Has("validate"), RawExport: string(ar.InputData), @@ -180,8 +180,7 @@ func handleImportProfile(ar *api.Request) (any, error) { AllowReplace: q.Has("allowReplace"), } } else { - request = &ProfileImportRequest{} - if err := json.Unmarshal(ar.InputData, request); err != nil { + if err := json.Unmarshal(ar.InputData, &request); err != nil { return nil, fmt.Errorf("%w: failed to parse import request: %w", ErrInvalidImportRequest, err) } } @@ -204,7 +203,7 @@ func handleImportProfile(ar *api.Request) (any, error) { } // Import. - return ImportProfile(request, profile.SourceLocal) + return ImportProfile(&request, profile.SourceLocal) } // ExportProfile exports a profile. From 58443631c4636865cecc9c46b109fc8eb0ffef08 Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Mon, 20 Nov 2023 12:15:00 +0100 Subject: [PATCH 3/5] Fix new profile icon API --- profile/api.go | 13 +++++++++---- profile/icons.go | 10 +++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/profile/api.go b/profile/api.go index 6b0a6d94..51f877da 100644 --- a/profile/api.go +++ b/profile/api.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "strings" "github.com/safing/portbase/api" @@ -26,7 +27,7 @@ func registerAPIEndpoints() error { if err := api.RegisterEndpoint(api.Endpoint{ Name: "Get Profile Icon", Description: "Returns the requested profile icon.", - Path: "profile/icon/{id:[0-9a-f]{40-80}}.{ext:[a-z]{3-4}}", + Path: "profile/icon/{id:[a-f0-9]*\\.[a-z]{3,4}}", Read: api.PermitUser, BelongsTo: module, DataFunc: handleGetProfileIcon, @@ -37,7 +38,7 @@ func registerAPIEndpoints() error { if err := api.RegisterEndpoint(api.Endpoint{ Name: "Update Profile Icon", Description: "Updates a profile icon.", - Path: "profile/icon/update", + Path: "profile/icon", Write: api.PermitUser, BelongsTo: module, StructFunc: handleUpdateProfileIcon, @@ -92,14 +93,18 @@ func handleMergeProfiles(ar *api.Request) (i interface{}, err error) { } func handleGetProfileIcon(ar *api.Request) (data []byte, err error) { + name := ar.URLVars["id"] + + ext := filepath.Ext(name) + // Get profile icon. - data, err = GetProfileIcon(ar.URLVars["id"], ar.URLVars["ext"]) + data, err = GetProfileIcon(name) if err != nil { return nil, err } // Set content type for icon. - contentType, ok := utils.MimeTypeByExtension(ar.URLVars["ext"]) + contentType, ok := utils.MimeTypeByExtension(ext) if ok { ar.ResponseHeader.Set("Content-Type", contentType) } diff --git a/profile/icons.go b/profile/icons.go index 6260d217..08355c9e 100644 --- a/profile/icons.go +++ b/profile/icons.go @@ -16,16 +16,20 @@ import ( var profileIconStoragePath = "" // GetProfileIcon returns the profile icon with the given ID and extension. -func GetProfileIcon(id, ext string) (data []byte, err error) { +func GetProfileIcon(name string) (data []byte, err error) { // Build storage path. - iconPath := filepath.Join(profileIconStoragePath, id+"."+ext) + iconPath := filepath.Clean( + filepath.Join(profileIconStoragePath, name), + ) + iconPath, err = filepath.Abs(iconPath) if err != nil { return nil, fmt.Errorf("failed to check icon path: %w", err) } + // Do a quick check if we are still within the right directory. // This check is not entirely correct, but is sufficient for this use case. - if !strings.HasPrefix(iconPath, profileIconStoragePath) { + if filepath.Dir(iconPath) != profileIconStoragePath { return nil, api.ErrorWithStatus(errors.New("invalid icon"), http.StatusBadRequest) } From bd988724c45707b93a76a48b5b2809110c5e97b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 22 Nov 2023 13:40:36 +0100 Subject: [PATCH 4/5] Add support for import/export of profile icon --- go.mod | 1 + go.sum | 2 + profile/icon.go | 72 +++++++++++++++++++++++++++ profile/icons.go | 5 ++ sync/profile.go | 108 ++++++++++++++++++++++------------------- sync/setting_single.go | 6 +-- sync/settings.go | 12 ++--- sync/util.go | 83 ++++++++++++++++++++++++++----- 8 files changed, 219 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index f7cf4ecd..e363d0dc 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 2bc4f40e..5f7320f8 100644 --- a/go.sum +++ b/go.sum @@ -260,6 +260,8 @@ github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608= github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= diff --git a/profile/icon.go b/profile/icon.go index 91a0ae58..96e2f3f0 100644 --- a/profile/icon.go +++ b/profile/icon.go @@ -1,9 +1,16 @@ package profile import ( + "errors" + "fmt" "strings" + "sync" + "github.com/vincent-petithory/dataurl" "golang.org/x/exp/slices" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" ) // Icon describes an icon. @@ -58,3 +65,68 @@ func sortAndCompactIcons(icons []Icon) []Icon { return icons } + +// GetIconAsDataURL returns the icon data as a data URL. +func (icon *Icon) GetIconAsDataURL() (bloburl string, err error) { + switch icon.Type { + case IconTypeFile: + return "", errors.New("getting icon from file is not supported") + + case IconTypeDatabase: + if !strings.HasPrefix(icon.Value, "cache:icons/") { + return "", errors.New("invalid icon db key") + } + r, err := iconDB.Get(icon.Value) + if err != nil { + return "", err + } + dbIcon, err := EnsureIconInDatabase(r) + if err != nil { + return "", err + } + return dbIcon.IconData, nil + + case IconTypeAPI: + data, err := GetProfileIcon(icon.Value) + if err != nil { + return "", err + } + return dataurl.EncodeBytes(data), nil + + default: + return "", errors.New("unknown icon type") + } +} + +var iconDB = database.NewInterface(&database.Options{ + Local: true, + Internal: true, +}) + +type IconInDatabase struct { + sync.Mutex + record.Base + + IconData string `json:"iconData,omitempty"` // DataURL +} + +// EnsureIconInDatabase ensures that the given record is a *IconInDatabase, and returns it. +func EnsureIconInDatabase(r record.Record) (*IconInDatabase, error) { + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + newIcon := &IconInDatabase{} + err := record.Unwrap(r, newIcon) + if err != nil { + return nil, err + } + return newIcon, nil + } + + // or adjust type + newIcon, ok := r.(*IconInDatabase) + if !ok { + return nil, fmt.Errorf("record not of type *IconInDatabase, but %T", r) + } + return newIcon, nil +} diff --git a/profile/icons.go b/profile/icons.go index 08355c9e..15ee316e 100644 --- a/profile/icons.go +++ b/profile/icons.go @@ -17,6 +17,11 @@ var profileIconStoragePath = "" // GetProfileIcon returns the profile icon with the given ID and extension. func GetProfileIcon(name string) (data []byte, err error) { + // Check if enabled. + if profileIconStoragePath == "" { + return nil, errors.New("api icon storage not configured") + } + // Build storage path. iconPath := filepath.Clean( filepath.Join(profileIconStoragePath, name), diff --git a/sync/profile.go b/sync/profile.go index 984e3cff..78112658 100644 --- a/sync/profile.go +++ b/sync/profile.go @@ -12,49 +12,49 @@ import ( "github.com/safing/portbase/config" "github.com/safing/portbase/log" "github.com/safing/portmaster/profile" + "github.com/vincent-petithory/dataurl" ) // ProfileExport holds an export of a profile. type ProfileExport struct { //nolint:maligned - Type Type `json:"type"` + Type Type `json:"type" yaml:"type"` // Identification - ID string `json:"id,omitempty"` - Source profile.ProfileSource `json:"source,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Source profile.ProfileSource `json:"source,omitempty" yaml:"source,omitempty"` // Human Metadata - Name string `json:"name"` - Description string `json:"description,omitempty"` - Homepage string `json:"homepage,omitempty"` - Icons []ProfileIcon `json:"icons,omitempty"` - PresentationPath string `json:"presPath,omitempty"` - UsePresentationPath bool `json:"usePresPath,omitempty"` + Name string `json:"name" yaml:"name"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Homepage string `json:"homepage,omitempty" yaml:"homepage,omitempty"` + PresentationPath string `json:"presPath,omitempty" yaml:"presPath,omitempty"` + UsePresentationPath bool `json:"usePresPath,omitempty" yaml:"usePresPath,omitempty"` + IconData string `json:"iconData,omitempty" yaml:"iconData,omitempty"` // DataURL // Process matching - Fingerprints []ProfileFingerprint `json:"fingerprints"` + Fingerprints []ProfileFingerprint `json:"fingerprints" yaml:"fingerprints"` // Settings - Config map[string]any `json:"config,omitempty"` + Config map[string]any `json:"config,omitempty" yaml:"config,omitempty"` // Metadata - LastEdited *time.Time `json:"lastEdited,omitempty"` - Created *time.Time `json:"created,omitempty"` - Internal bool `json:"internal,omitempty"` + LastEdited *time.Time `json:"lastEdited,omitempty" yaml:"lastEdited,omitempty"` + Created *time.Time `json:"created,omitempty" yaml:"created,omitempty"` + Internal bool `json:"internal,omitempty" yaml:"internal,omitempty"` } -// ProfileIcon represents a profile icon. +// ProfileIcon represents a profile icon only. type ProfileIcon struct { - Type profile.IconType `json:"type"` - Value string `json:"value"` + IconData string `json:"iconData,omitempty" yaml:"iconData,omitempty"` // DataURL } // ProfileFingerprint represents a profile fingerprint. type ProfileFingerprint struct { - Type string `json:"type"` - Key string `json:"key,omitempty"` - Operation string `json:"operation"` - Value string `json:"value"` - MergedFrom string `json:"mergedFrom,omitempty"` + Type string `json:"type" yaml:"type"` + Key string `json:"key,omitempty" yaml:"key,omitempty"` + Operation string `json:"operation" yaml:"operation"` + Value string `json:"value" yaml:"value"` + MergedFrom string `json:"mergedFrom,omitempty" yaml:"mergedFrom,omitempty"` } // ProfileExportRequest is a request for a profile export. @@ -124,7 +124,8 @@ func registerProfileAPI() error { Method: http.MethodPost, Field: "allowUnknown", Description: "Allow importing of unknown values.", - }}, + }, + }, BelongsTo: module, StructFunc: handleImportProfile, }); err != nil { @@ -161,7 +162,7 @@ func handleExportProfile(ar *api.Request) (data []byte, err error) { return nil, err } - return serializeExport(export, ar) + return serializeProfileExport(export, ar) } func handleImportProfile(ar *api.Request) (any, error) { @@ -230,7 +231,6 @@ func ExportProfile(scopedID string) (*ProfileExport, error) { Name: p.Name, Description: p.Description, Homepage: p.Homepage, - Icons: convertIconsToExport(p.Icons), PresentationPath: p.PresentationPath, UsePresentationPath: p.UsePresentationPath, @@ -253,6 +253,22 @@ func ExportProfile(scopedID string) (*ProfileExport, error) { export.Created = &created } + // Add first exportable icon to export. + if len(p.Icons) > 0 { + var err error + for _, icon := range p.Icons { + var iconDataURL string + iconDataURL, err = icon.GetIconAsDataURL() + if err == nil { + export.IconData = iconDataURL + break + } + } + if err != nil { + return nil, fmt.Errorf("%w: failed to export icon: %w", ErrExportFailed, err) + } + } + return export, nil } @@ -272,9 +288,8 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil profileID := profile.DeriveProfileID(fingerprints) if r.Export.ID != "" && r.Export.ID != profileID { return nil, ErrMismatch - } else { - r.Export.ID = profileID } + r.Export.ID = profileID // Check Fingerprints. _, err := profile.ParseFingerprints(fingerprints, "") if err != nil { @@ -349,7 +364,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil Name: in.Name, Description: in.Description, Homepage: in.Homepage, - Icons: convertIconsToInternal(in.Icons), PresentationPath: in.PresentationPath, UsePresentationPath: in.UsePresentationPath, @@ -370,6 +384,22 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil p.Created = in.Created.Unix() } + // Add icon to profile, if set. + if in.IconData != "" { + du, err := dataurl.DecodeString(in.IconData) + if err != nil { + return nil, fmt.Errorf("%w: icon data is invalid: %w", ErrImportFailed, err) + } + filename, err := profile.UpdateProfileIcon(du.Data, du.MediaType.Subtype) + if err != nil { + return nil, fmt.Errorf("%w: icon is invalid: %w", ErrImportFailed, err) + } + p.Icons = []profile.Icon{{ + Type: profile.IconTypeAPI, + Value: filename, + }} + } + // Save profile to db. p.SetKey(profile.MakeProfileKey(p.Source, p.ID)) err = p.Save() @@ -388,28 +418,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil return result, nil } -func convertIconsToExport(icons []profile.Icon) []ProfileIcon { - converted := make([]ProfileIcon, 0, len(icons)) - for _, icon := range icons { - converted = append(converted, ProfileIcon{ - Type: icon.Type, - Value: icon.Value, - }) - } - return converted -} - -func convertIconsToInternal(icons []ProfileIcon) []profile.Icon { - converted := make([]profile.Icon, 0, len(icons)) - for _, icon := range icons { - converted = append(converted, profile.Icon{ - Type: icon.Type, - Value: icon.Value, - }) - } - return converted -} - func convertFingerprintsToExport(fingerprints []profile.Fingerprint) []ProfileFingerprint { converted := make([]ProfileFingerprint, 0, len(fingerprints)) for _, fp := range fingerprints { diff --git a/sync/setting_single.go b/sync/setting_single.go index 32d43088..24cd0cbc 100644 --- a/sync/setting_single.go +++ b/sync/setting_single.go @@ -15,10 +15,10 @@ import ( // SingleSettingExport holds an export of a single setting. type SingleSettingExport struct { - Type Type `json:"type"` // Must be TypeSingleSetting - ID string `json:"id"` // Settings Key + Type Type `json:"type" yaml:"type"` // Must be TypeSingleSetting + ID string `json:"id" yaml:"id"` // Settings Key - Value any `json:"value"` + Value any `json:"value" yaml:"value"` } // SingleSettingImportRequest is a request to import a single setting. diff --git a/sync/settings.go b/sync/settings.go index ae033fa5..8f0dde12 100644 --- a/sync/settings.go +++ b/sync/settings.go @@ -15,25 +15,25 @@ import ( // SettingsExport holds an export of settings. type SettingsExport struct { - Type Type `json:"type"` + Type Type `json:"type" yaml:"type"` - Config map[string]any `json:"config"` + Config map[string]any `json:"config" yaml:"config"` } // SettingsImportRequest is a request to import settings. type SettingsImportRequest struct { - ImportRequest `json:",inline"` + ImportRequest `json:",inline" yaml:",inline"` // Reset all settings of target before import. // The ImportResult also reacts to this flag and correctly reports whether // any settings would be replaced or deleted. - Reset bool `json:"reset"` + Reset bool `json:"reset" yaml:"reset"` // AllowUnknown allows the import of unknown settings. // Otherwise, attempting to import an unknown setting will result in an error. - AllowUnknown bool `json:"allowUnknown"` + AllowUnknown bool `json:"allowUnknown" yaml:"allowUnknown"` - Export *SettingsExport `json:"export"` + Export *SettingsExport `json:"export" yaml:"export"` } func registerSettingsAPI() error { diff --git a/sync/util.go b/sync/util.go index bd343096..3d95b938 100644 --- a/sync/util.go +++ b/sync/util.go @@ -1,12 +1,16 @@ package sync import ( + "encoding/json" "errors" "fmt" "net/http" + yaml "gopkg.in/yaml.v3" + "github.com/safing/jess/filesig" "github.com/safing/portbase/api" + "github.com/safing/portbase/container" "github.com/safing/portbase/formats/dsd" ) @@ -101,28 +105,85 @@ var ( ) ) -func serializeExport(export any, ar *api.Request) ([]byte, error) { - // Serialize data. - data, mimeType, format, err := dsd.MimeDump(export, ar.Header.Get("Accept")) - if err != nil { - return nil, fmt.Errorf("failed to serialize data: %w", err) - } - ar.ResponseHeader.Set("Content-Type", mimeType) +func serializeExport(export any, ar *api.Request) (data []byte, err error) { + // Get format. + format := dsd.FormatFromAccept(ar.Header.Get("Accept")) - // Add checksum. + // Serialize and add checksum. switch format { case dsd.JSON: - data, err = filesig.AddJSONChecksum(data) + data, err = json.Marshal(export) + if err == nil { + data, err = filesig.AddJSONChecksum(data) + } case dsd.YAML: - data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom) + data, err = yaml.Marshal(export) + if err == nil { + data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom) + } default: return nil, dsd.ErrIncompatibleFormat } + if err != nil { + return nil, fmt.Errorf("failed to serialize: %w", err) + } + + // Set Content-Type HTTP Header. + ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[format]) + + return data, nil +} + +func serializeProfileExport(export *ProfileExport, ar *api.Request) ([]byte, error) { + // Do a regular serialize, if we don't need parts. + switch { + case export.IconData == "": + // With no icon, do a regular export. + return serializeExport(export, ar) + case dsd.FormatFromAccept(ar.Header.Get("Accept")) != dsd.YAML: + // Only export in parts for yaml. + return serializeExport(export, ar) + } + + // Step 1: Separate profile icon. + profileIconExport := &ProfileIcon{ + IconData: export.IconData, + } + export.IconData = "" + + // Step 2: Serialize main export. + profileData, err := yaml.Marshal(export) + if err != nil { + return nil, fmt.Errorf("failed to serialize profile data: %w", err) + } + + // Step 3: Serialize icon only. + iconData, err := yaml.Marshal(profileIconExport) + if err != nil { + return nil, fmt.Errorf("failed to serialize profile icon: %w", err) + } + + // Step 4: Stitch data together and add copyright notice for icon. + exportData := container.New( + profileData, + []byte(` +# The application icon below is the property of its respective owner. +# The icon is used for identification purposes only, and does not imply any endorsement or affiliation with their respective owners. +# It is the sole responsibility of the individual or entity sharing this dataset to ensure they have the necessary permissions to do so. +`), + iconData, + ).CompileData() + + // Step 4: Add checksum. + exportData, err = filesig.AddYAMLChecksum(exportData, filesig.TextPlacementBottom) if err != nil { return nil, fmt.Errorf("failed to add checksum: %w", err) } - return data, nil + // Set Content-Type HTTP Header. + ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[dsd.YAML]) + + return exportData, nil } func parseExport(request *ImportRequest, export any) error { From 8ac80c2a12ee54571c55534eb64b2415f0a0b692 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 22 Nov 2023 14:01:53 +0100 Subject: [PATCH 5/5] Fix linter warnings --- profile/api.go | 1 + profile/config.go | 2 +- profile/fingerprint.go | 11 ++++++----- profile/get.go | 2 +- profile/icon.go | 1 + profile/profile.go | 2 +- sync/profile.go | 3 ++- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/profile/api.go b/profile/api.go index 51f877da..1a69a4e4 100644 --- a/profile/api.go +++ b/profile/api.go @@ -116,6 +116,7 @@ type updateProfileIconResponse struct { Filename string `json:"filename"` } +//nolint:goconst func handleUpdateProfileIcon(ar *api.Request) (any, error) { // Check input. if len(ar.InputData) == 0 { diff --git a/profile/config.go b/profile/config.go index d65f10bd..fdfcd330 100644 --- a/profile/config.go +++ b/profile/config.go @@ -672,7 +672,7 @@ Current Features: - Disable Firefox' internal DNS-over-HTTPs resolver - Block direct access to public DNS resolvers -Please note that DNS bypass attempts might be additionally blocked in the Sytem D there too.`, +Please note that DNS bypass attempts might be additionally blocked in the System DNS Client App.`, OptType: config.OptTypeInt, ExpertiseLevel: config.ExpertiseLevelUser, ReleaseLevel: config.ReleaseLevelStable, diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 9caf8794..56e92172 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -163,7 +163,8 @@ func (fp fingerprintRegex) Match(value string) (score int) { return 0 } -type parsedFingerprints struct { +// ParsedFingerprints holds parsed fingerprints for fast usage. +type ParsedFingerprints struct { tagPrints []matchingFingerprint envPrints []matchingFingerprint pathPrints []matchingFingerprint @@ -171,8 +172,8 @@ type parsedFingerprints struct { } // ParseFingerprints parses the fingerprints to make them ready for matching. -func ParseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { - parsed = &parsedFingerprints{} +func ParseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *ParsedFingerprints, firstErr error) { + parsed = &ParsedFingerprints{} // Add deprecated LinkedPath to fingerprints, if they are empty. // TODO: Remove in v1.5 @@ -239,7 +240,7 @@ func ParseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed * return parsed, firstErr } -func (parsed *parsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchingPrint matchingFingerprint) { +func (parsed *ParsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchingPrint matchingFingerprint) { switch fp.Type { case FingerprintTypeTagID: parsed.tagPrints = append(parsed.tagPrints, matchingPrint) @@ -257,7 +258,7 @@ func (parsed *parsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchin // MatchFingerprints returns the highest matching score of the given // fingerprints and matching data. -func MatchFingerprints(prints *parsedFingerprints, md MatchingData) (highestScore int) { +func MatchFingerprints(prints *ParsedFingerprints, md MatchingData) (highestScore int) { // Check tags. tags := md.Tags() if len(tags) > 0 { diff --git a/profile/get.go b/profile/get.go index 2a5424cf..3fcf78e5 100644 --- a/profile/get.go +++ b/profile/get.go @@ -257,7 +257,7 @@ profileFeed: return profile, nil } -func loadProfileFingerprints(r record.Record) (parsed *parsedFingerprints, err error) { +func loadProfileFingerprints(r record.Record) (parsed *ParsedFingerprints, err error) { // Ensure it's a profile. profile, err := EnsureProfile(r) if err != nil { diff --git a/profile/icon.go b/profile/icon.go index 96e2f3f0..fe5f5c72 100644 --- a/profile/icon.go +++ b/profile/icon.go @@ -103,6 +103,7 @@ var iconDB = database.NewInterface(&database.Options{ Internal: true, }) +// IconInDatabase represents an icon saved to the database. type IconInDatabase struct { sync.Mutex record.Base diff --git a/profile/profile.go b/profile/profile.go index 87228acb..8cdb2f58 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -21,7 +21,7 @@ import ( ) // ProfileSource is the source of the profile. -type ProfileSource string +type ProfileSource string //nolint:golint // Profile Sources. const ( diff --git a/sync/profile.go b/sync/profile.go index 78112658..c6e936d5 100644 --- a/sync/profile.go +++ b/sync/profile.go @@ -8,11 +8,12 @@ import ( "strings" "time" + "github.com/vincent-petithory/dataurl" + "github.com/safing/portbase/api" "github.com/safing/portbase/config" "github.com/safing/portbase/log" "github.com/safing/portmaster/profile" - "github.com/vincent-petithory/dataurl" ) // ProfileExport holds an export of a profile.