From bd988724c45707b93a76a48b5b2809110c5e97b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 22 Nov 2023 13:40:36 +0100 Subject: [PATCH] Add support for import/export of profile icon --- go.mod | 1 + go.sum | 2 + profile/icon.go | 72 +++++++++++++++++++++++++++ profile/icons.go | 5 ++ sync/profile.go | 108 ++++++++++++++++++++++------------------- sync/setting_single.go | 6 +-- sync/settings.go | 12 ++--- sync/util.go | 83 ++++++++++++++++++++++++++----- 8 files changed, 219 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index f7cf4ecd..e363d0dc 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 2bc4f40e..5f7320f8 100644 --- a/go.sum +++ b/go.sum @@ -260,6 +260,8 @@ github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608= github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= diff --git a/profile/icon.go b/profile/icon.go index 91a0ae58..96e2f3f0 100644 --- a/profile/icon.go +++ b/profile/icon.go @@ -1,9 +1,16 @@ package profile import ( + "errors" + "fmt" "strings" + "sync" + "github.com/vincent-petithory/dataurl" "golang.org/x/exp/slices" + + "github.com/safing/portbase/database" + "github.com/safing/portbase/database/record" ) // Icon describes an icon. @@ -58,3 +65,68 @@ func sortAndCompactIcons(icons []Icon) []Icon { return icons } + +// GetIconAsDataURL returns the icon data as a data URL. +func (icon *Icon) GetIconAsDataURL() (bloburl string, err error) { + switch icon.Type { + case IconTypeFile: + return "", errors.New("getting icon from file is not supported") + + case IconTypeDatabase: + if !strings.HasPrefix(icon.Value, "cache:icons/") { + return "", errors.New("invalid icon db key") + } + r, err := iconDB.Get(icon.Value) + if err != nil { + return "", err + } + dbIcon, err := EnsureIconInDatabase(r) + if err != nil { + return "", err + } + return dbIcon.IconData, nil + + case IconTypeAPI: + data, err := GetProfileIcon(icon.Value) + if err != nil { + return "", err + } + return dataurl.EncodeBytes(data), nil + + default: + return "", errors.New("unknown icon type") + } +} + +var iconDB = database.NewInterface(&database.Options{ + Local: true, + Internal: true, +}) + +type IconInDatabase struct { + sync.Mutex + record.Base + + IconData string `json:"iconData,omitempty"` // DataURL +} + +// EnsureIconInDatabase ensures that the given record is a *IconInDatabase, and returns it. +func EnsureIconInDatabase(r record.Record) (*IconInDatabase, error) { + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + newIcon := &IconInDatabase{} + err := record.Unwrap(r, newIcon) + if err != nil { + return nil, err + } + return newIcon, nil + } + + // or adjust type + newIcon, ok := r.(*IconInDatabase) + if !ok { + return nil, fmt.Errorf("record not of type *IconInDatabase, but %T", r) + } + return newIcon, nil +} diff --git a/profile/icons.go b/profile/icons.go index 08355c9e..15ee316e 100644 --- a/profile/icons.go +++ b/profile/icons.go @@ -17,6 +17,11 @@ var profileIconStoragePath = "" // GetProfileIcon returns the profile icon with the given ID and extension. func GetProfileIcon(name string) (data []byte, err error) { + // Check if enabled. + if profileIconStoragePath == "" { + return nil, errors.New("api icon storage not configured") + } + // Build storage path. iconPath := filepath.Clean( filepath.Join(profileIconStoragePath, name), diff --git a/sync/profile.go b/sync/profile.go index 984e3cff..78112658 100644 --- a/sync/profile.go +++ b/sync/profile.go @@ -12,49 +12,49 @@ import ( "github.com/safing/portbase/config" "github.com/safing/portbase/log" "github.com/safing/portmaster/profile" + "github.com/vincent-petithory/dataurl" ) // ProfileExport holds an export of a profile. type ProfileExport struct { //nolint:maligned - Type Type `json:"type"` + Type Type `json:"type" yaml:"type"` // Identification - ID string `json:"id,omitempty"` - Source profile.ProfileSource `json:"source,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Source profile.ProfileSource `json:"source,omitempty" yaml:"source,omitempty"` // Human Metadata - Name string `json:"name"` - Description string `json:"description,omitempty"` - Homepage string `json:"homepage,omitempty"` - Icons []ProfileIcon `json:"icons,omitempty"` - PresentationPath string `json:"presPath,omitempty"` - UsePresentationPath bool `json:"usePresPath,omitempty"` + Name string `json:"name" yaml:"name"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Homepage string `json:"homepage,omitempty" yaml:"homepage,omitempty"` + PresentationPath string `json:"presPath,omitempty" yaml:"presPath,omitempty"` + UsePresentationPath bool `json:"usePresPath,omitempty" yaml:"usePresPath,omitempty"` + IconData string `json:"iconData,omitempty" yaml:"iconData,omitempty"` // DataURL // Process matching - Fingerprints []ProfileFingerprint `json:"fingerprints"` + Fingerprints []ProfileFingerprint `json:"fingerprints" yaml:"fingerprints"` // Settings - Config map[string]any `json:"config,omitempty"` + Config map[string]any `json:"config,omitempty" yaml:"config,omitempty"` // Metadata - LastEdited *time.Time `json:"lastEdited,omitempty"` - Created *time.Time `json:"created,omitempty"` - Internal bool `json:"internal,omitempty"` + LastEdited *time.Time `json:"lastEdited,omitempty" yaml:"lastEdited,omitempty"` + Created *time.Time `json:"created,omitempty" yaml:"created,omitempty"` + Internal bool `json:"internal,omitempty" yaml:"internal,omitempty"` } -// ProfileIcon represents a profile icon. +// ProfileIcon represents a profile icon only. type ProfileIcon struct { - Type profile.IconType `json:"type"` - Value string `json:"value"` + IconData string `json:"iconData,omitempty" yaml:"iconData,omitempty"` // DataURL } // ProfileFingerprint represents a profile fingerprint. type ProfileFingerprint struct { - Type string `json:"type"` - Key string `json:"key,omitempty"` - Operation string `json:"operation"` - Value string `json:"value"` - MergedFrom string `json:"mergedFrom,omitempty"` + Type string `json:"type" yaml:"type"` + Key string `json:"key,omitempty" yaml:"key,omitempty"` + Operation string `json:"operation" yaml:"operation"` + Value string `json:"value" yaml:"value"` + MergedFrom string `json:"mergedFrom,omitempty" yaml:"mergedFrom,omitempty"` } // ProfileExportRequest is a request for a profile export. @@ -124,7 +124,8 @@ func registerProfileAPI() error { Method: http.MethodPost, Field: "allowUnknown", Description: "Allow importing of unknown values.", - }}, + }, + }, BelongsTo: module, StructFunc: handleImportProfile, }); err != nil { @@ -161,7 +162,7 @@ func handleExportProfile(ar *api.Request) (data []byte, err error) { return nil, err } - return serializeExport(export, ar) + return serializeProfileExport(export, ar) } func handleImportProfile(ar *api.Request) (any, error) { @@ -230,7 +231,6 @@ func ExportProfile(scopedID string) (*ProfileExport, error) { Name: p.Name, Description: p.Description, Homepage: p.Homepage, - Icons: convertIconsToExport(p.Icons), PresentationPath: p.PresentationPath, UsePresentationPath: p.UsePresentationPath, @@ -253,6 +253,22 @@ func ExportProfile(scopedID string) (*ProfileExport, error) { export.Created = &created } + // Add first exportable icon to export. + if len(p.Icons) > 0 { + var err error + for _, icon := range p.Icons { + var iconDataURL string + iconDataURL, err = icon.GetIconAsDataURL() + if err == nil { + export.IconData = iconDataURL + break + } + } + if err != nil { + return nil, fmt.Errorf("%w: failed to export icon: %w", ErrExportFailed, err) + } + } + return export, nil } @@ -272,9 +288,8 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil profileID := profile.DeriveProfileID(fingerprints) if r.Export.ID != "" && r.Export.ID != profileID { return nil, ErrMismatch - } else { - r.Export.ID = profileID } + r.Export.ID = profileID // Check Fingerprints. _, err := profile.ParseFingerprints(fingerprints, "") if err != nil { @@ -349,7 +364,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil Name: in.Name, Description: in.Description, Homepage: in.Homepage, - Icons: convertIconsToInternal(in.Icons), PresentationPath: in.PresentationPath, UsePresentationPath: in.UsePresentationPath, @@ -370,6 +384,22 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil p.Created = in.Created.Unix() } + // Add icon to profile, if set. + if in.IconData != "" { + du, err := dataurl.DecodeString(in.IconData) + if err != nil { + return nil, fmt.Errorf("%w: icon data is invalid: %w", ErrImportFailed, err) + } + filename, err := profile.UpdateProfileIcon(du.Data, du.MediaType.Subtype) + if err != nil { + return nil, fmt.Errorf("%w: icon is invalid: %w", ErrImportFailed, err) + } + p.Icons = []profile.Icon{{ + Type: profile.IconTypeAPI, + Value: filename, + }} + } + // Save profile to db. p.SetKey(profile.MakeProfileKey(p.Source, p.ID)) err = p.Save() @@ -388,28 +418,6 @@ func ImportProfile(r *ProfileImportRequest, requiredProfileSource profile.Profil return result, nil } -func convertIconsToExport(icons []profile.Icon) []ProfileIcon { - converted := make([]ProfileIcon, 0, len(icons)) - for _, icon := range icons { - converted = append(converted, ProfileIcon{ - Type: icon.Type, - Value: icon.Value, - }) - } - return converted -} - -func convertIconsToInternal(icons []ProfileIcon) []profile.Icon { - converted := make([]profile.Icon, 0, len(icons)) - for _, icon := range icons { - converted = append(converted, profile.Icon{ - Type: icon.Type, - Value: icon.Value, - }) - } - return converted -} - func convertFingerprintsToExport(fingerprints []profile.Fingerprint) []ProfileFingerprint { converted := make([]ProfileFingerprint, 0, len(fingerprints)) for _, fp := range fingerprints { diff --git a/sync/setting_single.go b/sync/setting_single.go index 32d43088..24cd0cbc 100644 --- a/sync/setting_single.go +++ b/sync/setting_single.go @@ -15,10 +15,10 @@ import ( // SingleSettingExport holds an export of a single setting. type SingleSettingExport struct { - Type Type `json:"type"` // Must be TypeSingleSetting - ID string `json:"id"` // Settings Key + Type Type `json:"type" yaml:"type"` // Must be TypeSingleSetting + ID string `json:"id" yaml:"id"` // Settings Key - Value any `json:"value"` + Value any `json:"value" yaml:"value"` } // SingleSettingImportRequest is a request to import a single setting. diff --git a/sync/settings.go b/sync/settings.go index ae033fa5..8f0dde12 100644 --- a/sync/settings.go +++ b/sync/settings.go @@ -15,25 +15,25 @@ import ( // SettingsExport holds an export of settings. type SettingsExport struct { - Type Type `json:"type"` + Type Type `json:"type" yaml:"type"` - Config map[string]any `json:"config"` + Config map[string]any `json:"config" yaml:"config"` } // SettingsImportRequest is a request to import settings. type SettingsImportRequest struct { - ImportRequest `json:",inline"` + ImportRequest `json:",inline" yaml:",inline"` // Reset all settings of target before import. // The ImportResult also reacts to this flag and correctly reports whether // any settings would be replaced or deleted. - Reset bool `json:"reset"` + Reset bool `json:"reset" yaml:"reset"` // AllowUnknown allows the import of unknown settings. // Otherwise, attempting to import an unknown setting will result in an error. - AllowUnknown bool `json:"allowUnknown"` + AllowUnknown bool `json:"allowUnknown" yaml:"allowUnknown"` - Export *SettingsExport `json:"export"` + Export *SettingsExport `json:"export" yaml:"export"` } func registerSettingsAPI() error { diff --git a/sync/util.go b/sync/util.go index bd343096..3d95b938 100644 --- a/sync/util.go +++ b/sync/util.go @@ -1,12 +1,16 @@ package sync import ( + "encoding/json" "errors" "fmt" "net/http" + yaml "gopkg.in/yaml.v3" + "github.com/safing/jess/filesig" "github.com/safing/portbase/api" + "github.com/safing/portbase/container" "github.com/safing/portbase/formats/dsd" ) @@ -101,28 +105,85 @@ var ( ) ) -func serializeExport(export any, ar *api.Request) ([]byte, error) { - // Serialize data. - data, mimeType, format, err := dsd.MimeDump(export, ar.Header.Get("Accept")) - if err != nil { - return nil, fmt.Errorf("failed to serialize data: %w", err) - } - ar.ResponseHeader.Set("Content-Type", mimeType) +func serializeExport(export any, ar *api.Request) (data []byte, err error) { + // Get format. + format := dsd.FormatFromAccept(ar.Header.Get("Accept")) - // Add checksum. + // Serialize and add checksum. switch format { case dsd.JSON: - data, err = filesig.AddJSONChecksum(data) + data, err = json.Marshal(export) + if err == nil { + data, err = filesig.AddJSONChecksum(data) + } case dsd.YAML: - data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom) + data, err = yaml.Marshal(export) + if err == nil { + data, err = filesig.AddYAMLChecksum(data, filesig.TextPlacementBottom) + } default: return nil, dsd.ErrIncompatibleFormat } + if err != nil { + return nil, fmt.Errorf("failed to serialize: %w", err) + } + + // Set Content-Type HTTP Header. + ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[format]) + + return data, nil +} + +func serializeProfileExport(export *ProfileExport, ar *api.Request) ([]byte, error) { + // Do a regular serialize, if we don't need parts. + switch { + case export.IconData == "": + // With no icon, do a regular export. + return serializeExport(export, ar) + case dsd.FormatFromAccept(ar.Header.Get("Accept")) != dsd.YAML: + // Only export in parts for yaml. + return serializeExport(export, ar) + } + + // Step 1: Separate profile icon. + profileIconExport := &ProfileIcon{ + IconData: export.IconData, + } + export.IconData = "" + + // Step 2: Serialize main export. + profileData, err := yaml.Marshal(export) + if err != nil { + return nil, fmt.Errorf("failed to serialize profile data: %w", err) + } + + // Step 3: Serialize icon only. + iconData, err := yaml.Marshal(profileIconExport) + if err != nil { + return nil, fmt.Errorf("failed to serialize profile icon: %w", err) + } + + // Step 4: Stitch data together and add copyright notice for icon. + exportData := container.New( + profileData, + []byte(` +# The application icon below is the property of its respective owner. +# The icon is used for identification purposes only, and does not imply any endorsement or affiliation with their respective owners. +# It is the sole responsibility of the individual or entity sharing this dataset to ensure they have the necessary permissions to do so. +`), + iconData, + ).CompileData() + + // Step 4: Add checksum. + exportData, err = filesig.AddYAMLChecksum(exportData, filesig.TextPlacementBottom) if err != nil { return nil, fmt.Errorf("failed to add checksum: %w", err) } - return data, nil + // Set Content-Type HTTP Header. + ar.ResponseHeader.Set("Content-Type", dsd.FormatToMimeType[dsd.YAML]) + + return exportData, nil } func parseExport(request *ImportRequest, export any) error {