Finalize profile merging, add profile metadata state handling, re-attribute connections after profile deletion

profile/api.go

@@ -0,0 +1,66 @@
+package profile
+
+import (
+	"fmt"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/formats/dsd"
+)
+
+func registerAPIEndpoints() error {
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Merge profiles",
+		Description: "Merge multiple profiles into a new one.",
+		Path:        "profile/merge",
+		Write:       api.PermitUser,
+		BelongsTo:   module,
+		StructFunc:  handleMergeProfiles,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type mergeProfilesRequest struct {
+	Name string   `json:"name"` // Name of the new merged profile.
+	To   string   `json:"to"`   // Profile scoped ID.
+	From []string `json:"from"` // Profile scoped IDs.
+}
+
+type mergeprofilesResponse struct {
+	New string `json:"new"` // Profile scoped ID.
+}
+
+func handleMergeProfiles(ar *api.Request) (i interface{}, err error) {
+	request := &mergeProfilesRequest{}
+	_, err = dsd.MimeLoad(ar.InputData, ar.Header.Get("Content-Type"), request)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse request: %w", err)
+	}
+
+	// Get all profiles.
+	var (
+		primary     *Profile
+		secondaries = make([]*Profile, 0, len(request.From))
+	)
+	if primary, err = getProfile(request.To); err != nil {
+		return nil, fmt.Errorf("failed to get profile %s: %w", request.To, err)
+	}
+	for _, from := range request.From {
+		sp, err := getProfile(from)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get profile %s: %w", request.To, err)
+		}
+		secondaries = append(secondaries, sp)
+	}
+
+	newProfile, err := MergeProfiles(request.Name, primary, secondaries...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to merge profiles: %w", err)
+	}
+
+	return &mergeprofilesResponse{
+		New: newProfile.ScopedID(),
+	}, nil
+}

profile/database.go

@@ -16,6 +16,7 @@ import (
 // core:profiles/<scope>/<id>
 // cache:profiles/index/<identifier>/<value>
 
+// ProfilesDBPath is the base database path for profiles.
 const ProfilesDBPath = "core:profiles/"
 
 var profileDB = database.NewInterface(&database.Options{
@@ -59,8 +60,14 @@ func startProfileUpdateChecker() error {
 				}
 
 				// Get active profile.
-				activeProfile := getActiveProfile(strings.TrimPrefix(r.Key(), ProfilesDBPath))
+				scopedID := strings.TrimPrefix(r.Key(), ProfilesDBPath)
+				activeProfile := getActiveProfile(scopedID)
 				if activeProfile == nil {
+					// Check if profile is being deleted.
+					if r.Meta().IsDeleted() {
+						meta.MarkDeleted(scopedID)
+					}
+
 					// Don't do any additional actions if the profile is not active.
 					continue profileFeed
 				}
@@ -74,7 +81,9 @@ func startProfileUpdateChecker() error {
 				// Always mark as outdated if the record is being deleted.
 				if r.Meta().IsDeleted() {
 					activeProfile.outdated.Set()
-					module.TriggerEvent(profileConfigChange, nil)
+
+					meta.MarkDeleted(scopedID)
+					module.TriggerEvent(DeletedEvent, scopedID)
 					continue
 				}
 
@@ -83,7 +92,7 @@ func startProfileUpdateChecker() error {
 				receivedProfile, err := EnsureProfile(r)
 				if err != nil || !receivedProfile.savedInternally {
 					activeProfile.outdated.Set()
-					module.TriggerEvent(profileConfigChange, nil)
+					module.TriggerEvent(ConfigChangeEvent, scopedID)
 				}
 			case <-ctx.Done():
 				return nil
@@ -105,6 +114,11 @@ func (h *databaseHook) UsesPrePut() bool {
 
 // PrePut implements the Hook interface.
 func (h *databaseHook) PrePut(r record.Record) (record.Record, error) {
+	// Do not intervene with metadata key.
+	if r.Key() == profilesMetadataKey {
+		return r, nil
+	}
+
 	// convert
 	profile, err := EnsureProfile(r)
 	if err != nil {

profile/endpoints/endpoint.go

@@ -202,7 +202,8 @@ func invalidDefinitionError(fields []string, msg string) error {
 	return fmt.Errorf(`invalid endpoint definition: "%s" - %s`, strings.Join(fields, " "), msg)
 }
 
-func parseEndpoint(value string) (endpoint Endpoint, err error) { //nolint:gocognit
+//nolint:gocognit,nakedret
+func parseEndpoint(value string) (endpoint Endpoint, err error) {
 	fields := strings.Fields(value)
 	if len(fields) < 2 {
 		return nil, fmt.Errorf(`invalid endpoint definition: "%s"`, value)

profile/get.go

@@ -287,6 +287,9 @@ func loadProfile(r record.Record) (*Profile, error) {
 	// Set saved internally to suppress outdating profiles if saving internally.
 	profile.savedInternally = true
 
+	// Mark as recently seen.
+	meta.UpdateLastSeen(profile.ScopedID())
+
 	// return parsed profile
 	return profile, nil
 }

profile/merge.go

@@ -1,8 +1,10 @@
 package profile
 
 import (
+	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/safing/portbase/database/record"
 )
@@ -11,19 +13,42 @@ import (
 // The new profile is saved and returned.
 // Only the icon and fingerprints are inherited from other profiles.
 // All other information is taken only from the primary profile.
-func MergeProfiles(primary *Profile, secondaries ...*Profile) (newProfile *Profile, err error) {
+func MergeProfiles(name string, primary *Profile, secondaries ...*Profile) (newProfile *Profile, err error) {
+	if primary == nil || len(secondaries) == 0 {
+		return nil, errors.New("must supply both a primary and at least one secondary profile for merging")
+	}
+
 	// Fill info from primary profile.
+	nowUnix := time.Now().Unix()
 	newProfile = &Profile{
 		Base:                record.Base{},
 		RWMutex:             sync.RWMutex{},
 		ID:                  "", // Omit ID to derive it from the new fingerprints.
 		Source:              primary.Source,
-		Name:                primary.Name,
+		Name:                name,
 		Description:         primary.Description,
 		Homepage:            primary.Homepage,
 		UsePresentationPath: false, // Disable presentation path.
 		SecurityLevel:       primary.SecurityLevel,
 		Config:              primary.Config,
+		Created:             nowUnix,
+	}
+
+	// Fall back to name of primary profile, if none is set.
+	if newProfile.Name == "" {
+		newProfile.Name = primary.Name
+	}
+
+	// If any profile was edited, set LastEdited to now.
+	if primary.LastEdited > 0 {
+		newProfile.LastEdited = nowUnix
+	} else {
+		for _, sp := range secondaries {
+			if sp.LastEdited > 0 {
+				newProfile.LastEdited = nowUnix
+				break
+			}
+		}
 	}
 
 	// Collect all icons.
@@ -35,7 +60,7 @@ func MergeProfiles(primary *Profile, secondaries ...*Profile) (newProfile *Profi
 	newProfile.Icons = sortAndCompactIcons(newProfile.Icons)
 
 	// Collect all fingerprints.
-	newProfile.Fingerprints = make([]Fingerprint, 0, len(secondaries)+1) // Guess the needed space.
+	newProfile.Fingerprints = make([]Fingerprint, 0, len(primary.Fingerprints)+len(secondaries)) // Guess the needed space.
 	newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, primary.Fingerprints, primary.ScopedID())
 	for _, sp := range secondaries {
 		newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, sp.Fingerprints, sp.ScopedID())
@@ -44,26 +69,19 @@ func MergeProfiles(primary *Profile, secondaries ...*Profile) (newProfile *Profi
 
 	// Save new profile.
 	newProfile = New(newProfile)
-	err = newProfile.Save()
-	if err != nil {
+	if err := newProfile.Save(); err != nil {
 		return nil, fmt.Errorf("failed to save merged profile: %w", err)
 	}
-	// FIXME: Should we ... ?
-	// newProfile.updateMetadata()
-	// newProfile.updateMetadataFromSystem()
 
 	// Delete all previous profiles.
-	// FIXME:
-	/*
-		primary.Meta().Delete()
-		// Set as outdated and remove from active profiles.
-		// Signify that profile was deleted and save for sync.
-		for _, sp := range secondaries {
-			sp.Meta().Delete()
-			// Set as outdated and remove from active profiles.
-			// Signify that profile was deleted and save for sync.
+	if err := primary.delete(); err != nil {
+		return nil, fmt.Errorf("failed to delete primary profile %s: %w", primary.ScopedID(), err)
+	}
+	for _, sp := range secondaries {
+		if err := sp.delete(); err != nil {
+			return nil, fmt.Errorf("failed to delete secondary profile %s: %w", sp.ScopedID(), err)
 		}
-	*/
+	}
 
 	return newProfile, nil
 }

profile/meta.go

@@ -0,0 +1,184 @@
+package profile
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/safing/portbase/database/record"
+)
+
+// ProfilesMetadata holds metadata about all profiles that are not fit to be
+// stored with the profiles themselves.
+type ProfilesMetadata struct {
+	record.Base
+	sync.Mutex
+
+	States map[string]*MetaState
+}
+
+// MetaState describes the state of a profile.
+type MetaState struct {
+	State string
+	At    time.Time
+}
+
+// Profile metadata states.
+const (
+	MetaStateSeen    = "seen"
+	MetaStateDeleted = "deleted"
+)
+
+// EnsureProfilesMetadata ensures that the given record is a *ProfilesMetadata, and returns it.
+func EnsureProfilesMetadata(r record.Record) (*ProfilesMetadata, error) {
+	// unwrap
+	if r.IsWrapped() {
+		// only allocate a new struct, if we need it
+		newMeta := &ProfilesMetadata{}
+		err := record.Unwrap(r, newMeta)
+		if err != nil {
+			return nil, err
+		}
+		return newMeta, nil
+	}
+
+	// or adjust type
+	newMeta, ok := r.(*ProfilesMetadata)
+	if !ok {
+		return nil, fmt.Errorf("record not of type *Profile, but %T", r)
+	}
+	return newMeta, nil
+}
+
+var (
+	profilesMetadataKey = ProfilesDBPath + "meta"
+
+	meta *ProfilesMetadata
+
+	removeDeletedEntriesAfter = 30 * 24 * time.Hour
+)
+
+// loadProfilesMetadata loads the profile metadata from the database.
+// It may only be called during module starting, as there is no lock for "meta" itself.
+func loadProfilesMetadata() error {
+	r, err := profileDB.Get(profilesMetadataKey)
+	if err != nil {
+		return err
+	}
+	loadedMeta, err := EnsureProfilesMetadata(r)
+	if err != nil {
+		return err
+	}
+
+	// Set package variable.
+	meta = loadedMeta
+	return nil
+}
+
+func (meta *ProfilesMetadata) check() {
+	if meta.States == nil {
+		meta.States = make(map[string]*MetaState)
+	}
+}
+
+// Save saves the profile metadata to the database.
+func (meta *ProfilesMetadata) Save() error {
+	if meta == nil {
+		return nil
+	}
+
+	func() {
+		meta.Lock()
+		defer meta.Unlock()
+
+		if !meta.KeyIsSet() {
+			meta.SetKey(profilesMetadataKey)
+		}
+	}()
+
+	meta.Clean()
+	return profileDB.Put(meta)
+}
+
+// Clean removes old entries.
+func (meta *ProfilesMetadata) Clean() {
+	if meta == nil {
+		return
+	}
+
+	meta.Lock()
+	defer meta.Unlock()
+
+	for key, state := range meta.States {
+		switch {
+		case state == nil:
+			delete(meta.States, key)
+		case state.State != MetaStateDeleted:
+			continue
+		case time.Since(state.At) > removeDeletedEntriesAfter:
+			delete(meta.States, key)
+		}
+	}
+}
+
+// GetLastSeen returns when the profile with the given ID was last seen.
+func (meta *ProfilesMetadata) GetLastSeen(scopedID string) *time.Time {
+	if meta == nil {
+		return nil
+	}
+
+	meta.Lock()
+	defer meta.Unlock()
+
+	state := meta.States[scopedID]
+	switch {
+	case state == nil:
+		return nil
+	case state.State == MetaStateSeen:
+		return &state.At
+	default:
+		return nil
+	}
+}
+
+// UpdateLastSeen sets the profile with the given ID as last seen now.
+func (meta *ProfilesMetadata) UpdateLastSeen(scopedID string) {
+	if meta == nil {
+		return
+	}
+
+	meta.Lock()
+	defer meta.Unlock()
+
+	meta.States[scopedID] = &MetaState{
+		State: MetaStateSeen,
+		At:    time.Now().UTC(),
+	}
+}
+
+// MarkDeleted marks the profile with the given ID as deleted.
+func (meta *ProfilesMetadata) MarkDeleted(scopedID string) {
+	if meta == nil {
+		return
+	}
+
+	meta.Lock()
+	defer meta.Unlock()
+
+	meta.States[scopedID] = &MetaState{
+		State: MetaStateDeleted,
+		At:    time.Now().UTC(),
+	}
+}
+
+// RemoveState removes any state of the profile with the given ID.
+func (meta *ProfilesMetadata) RemoveState(scopedID string) {
+	if meta == nil {
+		return
+	}
+
+	meta.Lock()
+	defer meta.Unlock()
+
+	delete(meta.States, scopedID)
+}

profile/migrations.go

@@ -32,11 +32,11 @@ func registerMigrations() error {
 			Version:     "v1.4.7",
 			MigrateFunc: migrateIcons,
 		},
-		// migration.Migration{
-		// 	Description: "Migrate from random profile IDs to fingerprint-derived IDs",
-		// 	Version:     "v1.5.1",
-		// 	MigrateFunc: migrateToDerivedIDs,
-		// },
+		migration.Migration{
+			Description: "Migrate from random profile IDs to fingerprint-derived IDs",
+			Version:     "v1.5.0",
+			MigrateFunc: migrateToDerivedIDs,
+		},
 	)
 }
 

profile/module.go

@@ -1,8 +1,10 @@
 package profile
 
 import (
+	"errors"
 	"os"
 
+	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/database/migration"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
@@ -16,13 +18,16 @@ var (
 	updatesPath string
 )
 
+// Events.
 const (
-	profileConfigChange = "profile config change"
+	ConfigChangeEvent = "profile config change"
+	DeletedEvent      = "profile deleted"
 )
 
 func init() {
-	module = modules.Register("profiles", prep, start, nil, "base", "updates")
-	module.RegisterEvent(profileConfigChange, true)
+	module = modules.Register("profiles", prep, start, stop, "base", "updates")
+	module.RegisterEvent(ConfigChangeEvent, true)
+	module.RegisterEvent(DeletedEvent, true)
 }
 
 func prep() error {
@@ -47,6 +52,14 @@ func start() error {
 		updatesPath += string(os.PathSeparator)
 	}
 
+	if err := loadProfilesMetadata(); err != nil {
+		if !errors.Is(err, database.ErrNotFound) {
+			log.Warningf("profile: failed to load profiles metadata, falling back to empty state: %s", err)
+		}
+		meta = &ProfilesMetadata{}
+	}
+	meta.check()
+
 	if err := migrations.Migrate(module.Ctx); err != nil {
 		log.Errorf("profile: migrations failed: %s", err)
 	}
@@ -73,5 +86,13 @@ func start() error {
 		log.Warningf("profile: error during loading global profile from configuration: %s", err)
 	}
 
+	if err := registerAPIEndpoints(); err != nil {
+		return err
+	}
+
 	return nil
 }
+
+func stop() error {
+	return meta.Save()
+}

profile/profile.go

@@ -304,6 +304,24 @@ func (profile *Profile) Save() error {
 	return profileDB.Put(profile)
 }
 
+// delete deletes the profile from the database.
+func (profile *Profile) delete() error {
+	// Check if a key is set.
+	if !profile.KeyIsSet() {
+		return errors.New("key is not set")
+	}
+
+	// Delete from database.
+	profile.Meta().Delete()
+	err := profileDB.Put(profile)
+	if err != nil {
+		return err
+	}
+
+	// Post handling is done by the profile update feed.
+	return nil
+}
+
 // MarkStillActive marks the profile as still active.
 func (profile *Profile) MarkStillActive() {
 	atomic.StoreInt64(profile.lastActive, time.Now().Unix())