From 59f776ce2f4dcb27b5371b0b20989c1b79396cd5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 3 Oct 2022 22:10:43 +0200 Subject: [PATCH 01/15] Move from LinkedPath to Fingerprints and PresentationPath --- profile/active.go | 15 -- profile/config-update.go | 17 +- profile/database.go | 36 +--- profile/fingerprint.go | 323 ++++++++++++++++++++++++++++ profile/get.go | 297 ++++++++++++++++--------- profile/profile-layered-provider.go | 8 +- profile/profile-layered.go | 16 +- profile/profile.go | 95 ++++---- profile/special.go | 145 ++++++++++--- 9 files changed, 699 insertions(+), 253 deletions(-) create mode 100644 profile/fingerprint.go diff --git a/profile/active.go b/profile/active.go index 94c86603..6de5041d 100644 --- a/profile/active.go +++ b/profile/active.go @@ -38,21 +38,6 @@ func getAllActiveProfiles() []*Profile { return result } -// findActiveProfile searched for an active local profile using the linked path. -func findActiveProfile(linkedPath string) *Profile { - activeProfilesLock.RLock() - defer activeProfilesLock.RUnlock() - - for _, activeProfile := range activeProfiles { - if activeProfile.LinkedPath == linkedPath { - activeProfile.MarkStillActive() - return activeProfile - } - } - - return nil -} - // addActiveProfile registers a active profile. func addActiveProfile(profile *Profile) { activeProfilesLock.Lock() diff --git a/profile/config-update.go b/profile/config-update.go index f8979d3d..6f249548 100644 --- a/profile/config-update.go +++ b/profile/config-update.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/safing/portbase/config" "github.com/safing/portbase/modules" "github.com/safing/portmaster/intel/filterlists" "github.com/safing/portmaster/profile/endpoints" @@ -91,11 +90,7 @@ func updateGlobalConfigProfile(ctx context.Context, task *modules.Task) error { lastErr = err } - // build global profile for reference - profile := New(SourceSpecial, "global-config", "", nil) - profile.Name = "Global Configuration" - profile.Internal = true - + // Build config. newConfig := make(map[string]interface{}) // fill profile config options for key, value := range cfgStringOptions { @@ -111,8 +106,14 @@ func updateGlobalConfigProfile(ctx context.Context, task *modules.Task) error { newConfig[key] = value() } - // expand and assign - profile.Config = config.Expand(newConfig) + // Build global profile for reference. + profile := New(&Profile{ + ID: "global-config", + Source: SourceSpecial, + Name: "Global Configuration", + Config: newConfig, + Internal: true, + }) // save profile err = profile.Save() diff --git a/profile/database.go b/profile/database.go index 2438d667..c59235fd 100644 --- a/profile/database.go +++ b/profile/database.go @@ -65,41 +65,19 @@ func startProfileUpdateChecker() error { continue profileFeed } - // If the record is being deleted, reset the profile. - // create an empty profile instead. - if r.Meta().IsDeleted() { - newProfile, err := GetProfile( - activeProfile.Source, - activeProfile.ID, - activeProfile.LinkedPath, - true, - ) - if err != nil { - log.Errorf("profile: failed to create new profile after reset: %s", err) - } else { - // Copy metadata from the old profile. - newProfile.copyMetadataFrom(activeProfile) - // Save the new profile. - err = newProfile.Save() - if err != nil { - log.Errorf("profile: failed to save new profile after reset: %s", err) - } - } - - // If the new profile was successfully created, update layered profile. - activeProfile.outdated.Set() - if err == nil { - newProfile.layeredProfile.Update() - } - module.TriggerEvent(profileConfigChange, nil) - } - // Always increase the revision counter of the layer profile. // This marks previous connections in the UI as decided with outdated settings. if activeProfile.layeredProfile != nil { activeProfile.layeredProfile.increaseRevisionCounter(true) } + // Always mark as outdated if the record is being deleted. + if r.Meta().IsDeleted() { + activeProfile.outdated.Set() + module.TriggerEvent(profileConfigChange, nil) + continue + } + // If the profile is saved externally (eg. via the API), have the // next one to use it reload the profile from the database. receivedProfile, err := EnsureProfile(r) diff --git a/profile/fingerprint.go b/profile/fingerprint.go new file mode 100644 index 00000000..1fb44fbf --- /dev/null +++ b/profile/fingerprint.go @@ -0,0 +1,323 @@ +package profile + +import ( + "fmt" + "regexp" + "strings" +) + +// # Matching and Scores +// +// There are three levels: +// +// 1. Type: What matched? +// 1. Tag: 40.000 points +// 2. Env: 30.000 points +// 3. MatchingPath: 20.000 points +// 4. Path: 10.000 points +// 2. Operation: How was it mached? +// 1. Equals: 3.000 points +// 2. Prefix: 2.000 points +// 3. Regex: 1.000 points +// 3. How "strong" was the match? +// 1. Equals: Length of path (irrelevant) +// 2. Prefix: Length of prefix +// 3. Regex: 0 (we are not suicidal) + +// ms-store:Microsoft.One.Note + +// Path Match /path/to/file +// Tag MS-Store Match value +// Env Regex Key Value + +// Fingerprint Type IDs. +const ( + FingerprintTypeTagID = "tag" + FingerprintTypeEnvID = "env" + FingerprintTypePathID = "path" // Matches both MatchingPath and Path. + + FingerprintOperationEqualsID = "equals" + FingerprintOperationPrefixID = "prefix" + FingerprintOperationRegexID = "regex" + + tagMatchBaseScore = 40_000 + envMatchBaseScore = 30_000 + matchingPathMatchBaseScore = 20_000 + pathMatchBaseScore = 10_000 + + fingerprintEqualsBaseScore = 3_000 + fingerprintPrefixBaseScore = 2_000 + fingerprintRegexBaseScore = 1_000 + + maxMatchStrength = 499 +) + +type ( + // Fingerprint defines a way of matching a process. + // The Key is only valid - but required - for some types. + Fingerprint struct { + Type string + Key string // Key must always fully match. + Operation string + Value string + } + + // Tag represents a simple key/value kind of tag used in process metadata + // and fingerprints. + Tag struct { + Key string + Value string + } + + // MatchingData is an interface to fetching data in the matching process. + MatchingData interface { + Tags() []Tag + Env() map[string]string + Path() string + MatchingPath() string + } + + matchingFingerprint interface { + MatchesKey(key string) bool + Match(value string) (score int) + } +) + +// MatchesKey returns whether the optional fingerprint key (for some types +// only) matches the given key. +func (fp Fingerprint) MatchesKey(key string) bool { + return key == fp.Key +} + +// KeyInTags checks is the given key is in the tags. +func KeyInTags(tags []Tag, key string) bool { + for _, tag := range tags { + if key == tag.Key { + return true + } + } + return false +} + +// KeyAndValueInTags checks is the given key/value pair is in the tags. +func KeyAndValueInTags(tags []Tag, key, value string) bool { + for _, tag := range tags { + if key == tag.Key && value == tag.Value { + return true + } + } + return false +} + +type fingerprintEquals struct { + Fingerprint +} + +func (fp fingerprintEquals) Match(value string) (score int) { + if value == fp.Value { + return fingerprintEqualsBaseScore + checkMatchStrength(len(fp.Value)) + } + return 0 +} + +type fingerprintPrefix struct { + Fingerprint +} + +func (fp fingerprintPrefix) Match(value string) (score int) { + if strings.HasPrefix(value, fp.Value) { + return fingerprintPrefixBaseScore + checkMatchStrength(len(fp.Value)) + } + return 0 +} + +type fingerprintRegex struct { + Fingerprint + regex *regexp.Regexp +} + +func (fp fingerprintRegex) Match(value string) (score int) { + if fp.regex.MatchString(value) { + // Do not return any deviation from the base score. + // Trying to assign different scores to regex probably won't turn out to + // be a good idea. + return fingerprintRegexBaseScore + } + return 0 +} + +type parsedFingerprints struct { + tagPrints []matchingFingerprint + envPrints []matchingFingerprint + pathPrints []matchingFingerprint +} + +func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { + parsed = &parsedFingerprints{} + + // Add deprecated linked path to fingerprints. + if deprecatedLinkedPath != "" { + parsed.pathPrints = append(parsed.pathPrints, &fingerprintEquals{ + Fingerprint: Fingerprint{ + Type: FingerprintTypePathID, + Operation: FingerprintOperationEqualsID, + Value: deprecatedLinkedPath, + }, + }) + } + + // Parse all fingerprints. + // Do not fail when one fails, instead return the first encountered error. + for _, entry := range raw { + // Check type and required key. + switch entry.Type { + case FingerprintTypeTagID, FingerprintTypeEnvID: + if entry.Key == "" { + if firstErr == nil { + firstErr = fmt.Errorf("%s fingerprint is missing key", entry.Type) + } + continue + } + case FingerprintTypePathID: + // Don't need a key. + default: + // Unknown type. + if firstErr == nil { + firstErr = fmt.Errorf("unknown fingerprint type: %q", entry.Type) + } + continue + } + + // Create and/or collect operation match functions. + switch entry.Operation { + case FingerprintOperationEqualsID: + parsed.addMatchingFingerprint(entry, fingerprintEquals{entry}) + + case FingerprintOperationPrefixID: + parsed.addMatchingFingerprint(entry, fingerprintPrefix{entry}) + + case FingerprintOperationRegexID: + regex, err := regexp.Compile(entry.Value) + if err != nil { + if firstErr == nil { + firstErr = fmt.Errorf("failed to compile regex fingerprint: %s", entry.Value) + } + } else { + parsed.addMatchingFingerprint(entry, fingerprintRegex{ + Fingerprint: entry, + regex: regex, + }) + } + + default: + if firstErr == nil { + firstErr = fmt.Errorf("unknown fingerprint operation: %q", entry.Type) + } + } + } + + return parsed, firstErr +} + +func (parsed *parsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchingPrint matchingFingerprint) { + switch fp.Type { + case FingerprintTypeTagID: + parsed.tagPrints = append(parsed.tagPrints, matchingPrint) + case FingerprintTypeEnvID: + parsed.envPrints = append(parsed.envPrints, matchingPrint) + case FingerprintTypePathID: + parsed.pathPrints = append(parsed.pathPrints, matchingPrint) + default: + // This should never happen, as the types are checked already. + panic(fmt.Sprintf("unknown fingerprint type: %q", fp.Type)) + } +} + +// MatchFingerprints returns the highest matching score of the given +// fingerprints and matching data. +func MatchFingerprints(prints *parsedFingerprints, md MatchingData) (highestScore int) { + // Check tags. + for _, tagPrint := range prints.tagPrints { + for _, tag := range md.Tags() { + // Check if tag key matches. + if !tagPrint.MatchesKey(tag.Key) { + continue + } + + // Try matching the tag value. + score := tagPrint.Match(tag.Value) + if score > highestScore { + highestScore = score + } + } + } + // If something matched, add base score and return. + if highestScore > 0 { + return tagMatchBaseScore + highestScore + } + + // Check env. + for _, envPrint := range prints.envPrints { + for key, value := range md.Env() { + // Check if env key matches. + if !envPrint.MatchesKey(key) { + continue + } + + // Try matching the env value. + score := envPrint.Match(value) + if score > highestScore { + highestScore = score + } + } + } + // If something matched, add base score and return. + if highestScore > 0 { + return envMatchBaseScore + highestScore + } + + // Check matching path. + matchingPath := md.MatchingPath() + if matchingPath != "" { + for _, pathPrint := range prints.pathPrints { + // Try matching the path value. + score := pathPrint.Match(matchingPath) + if score > highestScore { + highestScore = score + } + } + // If something matched, add base score and return. + if highestScore > 0 { + return matchingPathMatchBaseScore + highestScore + } + } + + // Check path. + path := md.Path() + if path != "" { + for _, pathPrint := range prints.pathPrints { + // Try matching the path value. + score := pathPrint.Match(path) + if score > highestScore { + highestScore = score + } + } + // If something matched, add base score and return. + if highestScore > 0 { + return pathMatchBaseScore + highestScore + } + } + + // Nothing matched. + return 0 +} + +func checkMatchStrength(value int) int { + if value > maxMatchStrength { + return maxMatchStrength + } + if value < -maxMatchStrength { + return -maxMatchStrength + } + return value +} diff --git a/profile/get.go b/profile/get.go index c6ef0b5d..d9820c18 100644 --- a/profile/get.go +++ b/profile/get.go @@ -2,20 +2,24 @@ package profile import ( "errors" + "fmt" + "path" + "strings" "sync" - "github.com/safing/portbase/database" "github.com/safing/portbase/database/query" "github.com/safing/portbase/database/record" "github.com/safing/portbase/log" + "github.com/safing/portbase/notifications" ) var getProfileLock sync.Mutex -// GetProfile fetches a profile. This function ensures that the loaded profile -// is shared among all callers. You must always supply both the scopedID and -// linkedPath parameters whenever available. -func GetProfile(source profileSource, id, linkedPath string, reset bool) ( //nolint:gocognit +// GetLocalProfile fetches a profile. This function ensures that the loaded profile +// is shared among all callers. Always provide all available data points. +// Passing an ID without MatchingData is valid, but could lead to inconsistent +// data - use with caution. +func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *Profile) ( //nolint:gocognit profile *Profile, err error, ) { @@ -27,99 +31,90 @@ func GetProfile(source profileSource, id, linkedPath string, reset bool) ( //nol var previousVersion *Profile - // Fetch profile depending on the available information. - switch { - case id != "": - scopedID := makeScopedID(source, id) - - // Get profile via the scoped ID. - // Check if there already is an active and not outdated profile. - profile = getActiveProfile(scopedID) + // Get active profile based on the ID, if available. + if id != "" { + // Check if there already is an active profile. + profile = getActiveProfile(makeScopedID(SourceLocal, id)) if profile != nil { - profile.MarkStillActive() - - if profile.outdated.IsSet() || reset { - previousVersion = profile - } else { + // Mark active and return if not outdated. + if profile.outdated.IsNotSet() { + profile.MarkStillActive() return profile, nil } - } - // Get from database. - if !reset { - profile, err = getProfile(scopedID) - // Check if the profile is special and needs a reset. - if err == nil && specialProfileNeedsReset(profile) { - profile = getSpecialProfile(id, linkedPath) - } - } else { - // Simulate missing profile to create new one. - err = database.ErrNotFound + // If outdated, get from database. + previousVersion = profile + profile = nil } - - case linkedPath != "": - // Search for profile via a linked path. - // Check if there already is an active and not outdated profile for - // the linked path. - profile = findActiveProfile(linkedPath) - if profile != nil { - if profile.outdated.IsSet() || reset { - previousVersion = profile - } else { - return profile, nil - } - } - - // Get from database. - if !reset { - profile, err = findProfile(linkedPath) - // Check if the profile is special and needs a reset. - if err == nil && specialProfileNeedsReset(profile) { - profile = getSpecialProfile(id, linkedPath) - } - } else { - // Simulate missing profile to create new one. - err = database.ErrNotFound - } - - default: - return nil, errors.New("cannot fetch profile without ID or path") } - // Create new profile if none was found. - if errors.Is(err, database.ErrNotFound) { - err = nil + // In some cases, we might need to get a profile directly, without matching data. + // This could lead to inconsistent data - use with caution. + if md == nil { + if id == "" { + return nil, errors.New("cannot get local profiles without ID and matching data") + } - // Check if there is a special profile for this ID. - profile = getSpecialProfile(id, linkedPath) + profile, err = getProfile(makeScopedID(SourceLocal, id)) + if err != nil { + return nil, fmt.Errorf("failed to load profile %s by ID: %w", makeScopedID(SourceLocal, id), err) + } + } - // If not, create a standard profile. + // If we don't have a profile yet, find profile based on matching data. + if profile == nil { + profile, err = findProfile(SourceLocal, md) + if err != nil { + return nil, fmt.Errorf("failed to search for profile: %w", err) + } + } + + // If we still don't have a profile, create a new one. + if profile == nil { + // Try the profile creation callback, if we have one. + if createProfileCallback != nil { + profile = createProfileCallback() + } + + // If that did not work, create a standard profile. if profile == nil { - profile = New(SourceLocal, id, linkedPath, nil) + fpPath := md.MatchingPath() + if fpPath == "" { + fpPath = md.Path() + } + + profile = New(&Profile{ + ID: id, + Source: SourceLocal, + PresentationPath: md.Path(), + Fingerprints: []Fingerprint{ + { + Type: FingerprintTypePathID, + Operation: FingerprintOperationEqualsID, + Value: fpPath, + }, + }, + }) } } - // If there was a non-recoverable error, return here. - if err != nil { - return nil, err - } + // Prepare profile for first use. // Process profiles are coming directly from the database or are new. // As we don't use any caching, these will be new objects. - // Add a layeredProfile to local and network profiles. - if profile.Source == SourceLocal || profile.Source == SourceNetwork { - // If we are refetching, assign the layered profile from the previous version. - // The internal references will be updated when the layered profile checks for updates. - if previousVersion != nil { - profile.layeredProfile = previousVersion.layeredProfile - } + // Add a layeredProfile. - // Local profiles must have a layered profile, create a new one if it - // does not yet exist. - if profile.layeredProfile == nil { - profile.layeredProfile = NewLayeredProfile(profile) - } + // If we are refetching, assign the layered profile from the previous version. + // The internal references will be updated when the layered profile checks for updates. + if previousVersion != nil && previousVersion.layeredProfile != nil { + profile.layeredProfile = previousVersion.layeredProfile + } + + // Profiles must have a layered profile, create a new one if it + // does not yet exist. + if profile.layeredProfile == nil { + profile.layeredProfile = NewLayeredProfile(profile) } // Add the profile to the currently active profiles. @@ -137,40 +132,89 @@ func getProfile(scopedID string) (profile *Profile, err error) { } // Parse and prepare the profile, return the result. - return prepProfile(r) + return loadProfile(r) } // findProfile searches for a profile with the given linked path. If it cannot // find one, it will create a new profile for the given linked path. -func findProfile(linkedPath string) (profile *Profile, err error) { - // Search the database for a matching profile. - it, err := profileDB.Query( - query.New(makeProfileKey(SourceLocal, "")).Where( - query.Where("LinkedPath", query.SameAs, linkedPath), - ), +func findProfile(source profileSource, md MatchingData) (profile *Profile, err error) { + // TODO: Loading every profile from database and parsing it for every new + // process might be quite expensive. Measure impact and possibly improve. + + // Get iterator over all profiles. + it, err := profileDB.Query(query.New(profilesDBPath + makeScopedID(source, ""))) + if err != nil { + return nil, fmt.Errorf("failed to query for profiles: %w", err) + } + + // Find best matching profile. + var ( + highestScore int + bestMatch record.Record ) +profileFeed: + for r := range it.Next { + // Parse fingerprints. + prints, err := loadProfileFingerprints(r) + if err != nil { + log.Debugf("profile: failed to load fingerprints of %s: %s", r.Key(), err) + continue + } + + // Get matching score and compare. + score := MatchFingerprints(prints, md) + switch { + case score == 0: + // Continue to next. + case score > highestScore: + highestScore = score + bestMatch = r + case score == highestScore: + // Notify user of conflict and abort. + // Use first match - this should be consistent. + notifyConflictingProfiles(bestMatch, r, md) + it.Cancel() + break profileFeed + } + } + + // Check if there was an error while iterating. + if it.Err() != nil { + return nil, fmt.Errorf("failed to iterate over profiles: %w", err) + } + + // Return nothing if no profile matched. + if bestMatch == nil { + return nil, nil + } + + // If we have a match, parse and return the profile. + profile, err = loadProfile(bestMatch) + if err != nil { + return nil, fmt.Errorf("failed to parse selected profile %s: %w", bestMatch.Key(), err) + } + + // Check if this profile is already active and return the active version instead. + if activeProfile := getActiveProfile(profile.ScopedID()); activeProfile != nil { + return activeProfile, nil + } + + // Return nothing if no profile matched. + return profile, nil +} + +func loadProfileFingerprints(r record.Record) (parsed *parsedFingerprints, err error) { + // Ensure it's a profile. + profile, err := EnsureProfile(r) if err != nil { return nil, err } - // Only wait for the first result, or until the query ends. - r := <-it.Next - // Then cancel the query, should it still be running. - it.Cancel() - - // Prep and return an existing profile. - if r != nil { - profile, err = prepProfile(r) - return profile, err - } - - // If there was no profile in the database, create a new one, and return it. - profile = New(SourceLocal, "", linkedPath, nil) - - return profile, nil + // Parse and return fingerprints. + return parseFingerprints(profile.Fingerprints, profile.LinkedPath) } -func prepProfile(r record.Record) (*Profile, error) { +func loadProfile(r record.Record) (*Profile, error) { // ensure its a profile profile, err := EnsureProfile(r) if err != nil { @@ -192,3 +236,50 @@ func prepProfile(r record.Record) (*Profile, error) { // return parsed profile return profile, nil } + +func notifyConflictingProfiles(a, b record.Record, md MatchingData) { + // Get profile names. + var idA, nameA, idB, nameB string + profileA, err := EnsureProfile(a) + if err == nil { + idA = profileA.ScopedID() + nameA = profileA.Name + } else { + idA = strings.TrimPrefix(a.Key(), profilesDBPath) + nameA = path.Base(idA) + } + profileB, err := EnsureProfile(b) + if err == nil { + idB = profileB.ScopedID() + nameB = profileB.Name + } else { + idB = strings.TrimPrefix(b.Key(), profilesDBPath) + nameB = path.Base(idB) + } + + // Notify user about conflict. + notifications.NotifyWarn( + fmt.Sprintf("profiles:match-conflict:%s:%s", idA, idB), + "App Settings Match Conflict", + fmt.Sprintf( + "Multiple app settings match the app at %q with the same priority, please change on of them: %q or %q", + md.Path(), + nameA, + nameB, + ), + notifications.Action{ + Text: "Change (1)", + Type: notifications.ActionTypeOpenProfile, + Payload: idA, + }, + notifications.Action{ + Text: "Change (2)", + Type: notifications.ActionTypeOpenProfile, + Payload: idB, + }, + notifications.Action{ + ID: "ack", + Text: "OK", + }, + ) +} diff --git a/profile/profile-layered-provider.go b/profile/profile-layered-provider.go index 91378a40..81d54c4b 100644 --- a/profile/profile-layered-provider.go +++ b/profile/profile-layered-provider.go @@ -74,9 +74,11 @@ func getProfileRevision(p *Profile) (*LayeredProfile, error) { } // Update profiles if necessary. - if layeredProfile.NeedsUpdate() { - layeredProfile.Update() - } + // TODO: Cannot update as we have too little information. + // Just return the current state. Previous code: + // if layeredProfile.NeedsUpdate() { + // layeredProfile.Update() + // } return layeredProfile, nil } diff --git a/profile/profile-layered.go b/profile/profile-layered.go index 5fcbe75d..b2f7850b 100644 --- a/profile/profile-layered.go +++ b/profile/profile-layered.go @@ -57,8 +57,8 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile { lp := &LayeredProfile{ localProfile: localProfile, - layers: make([]*Profile, 0, len(localProfile.LinkedProfiles)+1), - LayerIDs: make([]string, 0, len(localProfile.LinkedProfiles)+1), + layers: make([]*Profile, 0, 1), + LayerIDs: make([]string, 0, 1), globalValidityFlag: config.NewValidityFlag(), RevisionCounter: 1, securityLevel: &securityLevelVal, @@ -246,17 +246,23 @@ func (lp *LayeredProfile) NeedsUpdate() (outdated bool) { } // Update checks for and replaces any outdated profiles. -func (lp *LayeredProfile) Update() (revisionCounter uint64) { +func (lp *LayeredProfile) Update(md MatchingData, createProfileCallback func() *Profile) (revisionCounter uint64) { lp.Lock() defer lp.Unlock() var changed bool for i, layer := range lp.layers { if layer.outdated.IsSet() { - changed = true + // Check for unsupported sources. + if layer.Source != SourceLocal { + log.Warningf("profile: updating profiles outside of local source is not supported: %s", layer.ScopedID()) + layer.outdated.UnSet() + continue + } // Update layer. - newLayer, err := GetProfile(layer.Source, layer.ID, layer.LinkedPath, false) + changed = true + newLayer, err := GetLocalProfile(layer.ID, md, createProfileCallback) if err != nil { log.Errorf("profiles: failed to update profile %s: %s", layer.ScopedID(), err) } else { diff --git a/profile/profile.go b/profile/profile.go index 62d7da3f..09dc8d68 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -26,11 +26,8 @@ type profileSource string // Profile Sources. const ( - SourceLocal profileSource = "local" // local, editable - SourceSpecial profileSource = "special" // specials (read-only) - SourceNetwork profileSource = "network" - SourceCommunity profileSource = "community" - SourceEnterprise profileSource = "enterprise" + SourceLocal profileSource = "local" // local, editable + SourceSpecial profileSource = "special" // specials (read-only) ) // Default Action IDs. @@ -83,11 +80,17 @@ type Profile struct { //nolint:maligned // not worth the effort Icon string // IconType describes the type of the Icon property. IconType iconType - // LinkedPath is a filesystem path to the executable this + // Deprecated: LinkedPath used to point to the executableis this // profile was created for. + // Until removed, it will be added to the Fingerprints as an exact path match. LinkedPath string // constant - // LinkedProfiles is a list of other profiles - LinkedProfiles []string + // PresentationPath holds the path of an executable that should be used for + // get representative information from, like the name of the program or the icon. + // Is automatically removed when the path does not exist. + // Is automatically populated with the next match when empty. + PresentationPath string + // Fingerprints holds process matching information. + Fingerprints []Fingerprint // SecurityLevel is the mininum security level to apply to // connections made with this profile. // Note(ppacher): we may deprecate this one as it can easily @@ -143,6 +146,11 @@ func (profile *Profile) prepProfile() { // prepare configuration profile.outdated = abool.New() profile.lastActive = new(int64) + + // Migration of LinkedPath to PresentationPath + if profile.PresentationPath == "" && profile.LinkedPath != "" { + profile.PresentationPath = profile.LinkedPath + } } func (profile *Profile) parseConfig() error { @@ -227,29 +235,25 @@ func (profile *Profile) parseConfig() error { // New returns a new Profile. // Optionally, you may supply custom configuration in the flat (key=value) form. -func New( - source profileSource, - id string, - linkedPath string, - customConfig map[string]interface{}, -) *Profile { - if customConfig != nil { - customConfig = config.Expand(customConfig) - } else { - customConfig = make(map[string]interface{}) +func New(profile *Profile) *Profile { + // Create profile if none is given. + if profile == nil { + profile = &Profile{} } - profile := &Profile{ - ID: id, - Source: source, - LinkedPath: linkedPath, - Created: time.Now().Unix(), - Config: customConfig, - savedInternally: true, + // Set default and internal values. + profile.Created = time.Now().Unix() + profile.savedInternally = true + + // Expand any given configuration. + if profile.Config != nil { + profile.Config = config.Expand(profile.Config) + } else { + profile.Config = make(map[string]interface{}) } // Generate random ID if none is given. - if id == "" { + if profile.ID == "" { profile.ID = utils.RandomUUID("").String() } @@ -437,13 +441,13 @@ func (profile *Profile) UpdateMetadata(binaryPath string) (changed bool) { var needsUpdateFromSystem bool // Check profile name. - filename := filepath.Base(profile.LinkedPath) + filename := filepath.Base(profile.PresentationPath) // Update profile name if it is empty or equals the filename, which is the // case for older profiles. if strings.TrimSpace(profile.Name) == "" || profile.Name == filename { // Generate a default profile name if does not exist. - profile.Name = osdetail.GenerateBinaryNameFromPath(profile.LinkedPath) + profile.Name = osdetail.GenerateBinaryNameFromPath(profile.PresentationPath) if profile.Name == filename { // TODO: Theoretically, the generated name could be identical to the // filename. @@ -462,37 +466,12 @@ func (profile *Profile) UpdateMetadata(binaryPath string) (changed bool) { return changed } -func (profile *Profile) copyMetadataFrom(otherProfile *Profile) (changed bool) { - if profile.Name != otherProfile.Name { - profile.Name = otherProfile.Name - changed = true - } - if profile.Description != otherProfile.Description { - profile.Description = otherProfile.Description - changed = true - } - if profile.Homepage != otherProfile.Homepage { - profile.Homepage = otherProfile.Homepage - changed = true - } - if profile.Icon != otherProfile.Icon { - profile.Icon = otherProfile.Icon - changed = true - } - if profile.IconType != otherProfile.IconType { - profile.IconType = otherProfile.IconType - changed = true - } - - return -} - // updateMetadataFromSystem updates the profile metadata with data from the // operating system and saves it afterwards. func (profile *Profile) updateMetadataFromSystem(ctx context.Context) error { // This function is only valid for local profiles. - if profile.Source != SourceLocal || profile.LinkedPath == "" { - return fmt.Errorf("tried to update metadata for non-local / non-linked profile %s", profile.ScopedID()) + if profile.Source != SourceLocal || profile.PresentationPath == "" { + return fmt.Errorf("tried to update metadata for non-local or non-path profile %s", profile.ScopedID()) } // Save the profile when finished, if needed. @@ -507,14 +486,14 @@ func (profile *Profile) updateMetadataFromSystem(ctx context.Context) error { }() // Get binary name from linked path. - newName, err := osdetail.GetBinaryNameFromSystem(profile.LinkedPath) + newName, err := osdetail.GetBinaryNameFromSystem(profile.PresentationPath) if err != nil { switch { case errors.Is(err, osdetail.ErrNotSupported): case errors.Is(err, osdetail.ErrNotFound): case errors.Is(err, osdetail.ErrEmptyOutput): default: - log.Warningf("profile: error while getting binary name for %s: %s", profile.LinkedPath, err) + log.Warningf("profile: error while getting binary name for %s: %s", profile.PresentationPath, err) } return nil } @@ -525,7 +504,7 @@ func (profile *Profile) updateMetadataFromSystem(ctx context.Context) error { } // Get filename of linked path for comparison. - filename := filepath.Base(profile.LinkedPath) + filename := filepath.Base(profile.PresentationPath) // TODO: Theoretically, the generated name from the system could be identical // to the filename. This would mean that the worker is triggered every time diff --git a/profile/special.go b/profile/special.go index c89ac3b1..89b06f3c 100644 --- a/profile/special.go +++ b/profile/special.go @@ -1,9 +1,12 @@ package profile import ( + "errors" "time" + "github.com/safing/portbase/database" "github.com/safing/portbase/log" + "github.com/safing/portmaster/status" ) const ( @@ -74,6 +77,73 @@ If you think you might have messed up the settings of the System DNS Client, jus PortmasterNotifierProfileDescription = `This is the Portmaster UI Tray Notifier.` ) +// GetSpecialProfile fetches a special profile. This function ensures that the loaded profile +// is shared among all callers. Always provide all available data points. +func GetSpecialProfile(id string, path string) ( //nolint:gocognit + profile *Profile, + err error, +) { + // Check if we have an ID. + if id == "" { + return nil, errors.New("cannot get special profile without ID") + } + scopedID := makeScopedID(SourceLocal, id) + + // Globally lock getting a profile. + // This does not happen too often, and it ensures we really have integrity + // and no race conditions. + getProfileLock.Lock() + defer getProfileLock.Unlock() + + // Check if there already is an active profile. + var previousVersion *Profile + profile = getActiveProfile(scopedID) + if profile != nil { + // Mark active and return if not outdated. + if profile.outdated.IsNotSet() { + profile.MarkStillActive() + return profile, nil + } + + // If outdated, get from database. + previousVersion = profile + } + + // Get special profile from DB and check if it needs a reset. + profile, err = getProfile(scopedID) + if err != nil { + if !errors.Is(err, database.ErrNotFound) { + log.Warningf("profile: failed to get special profile %s: %s", id, err) + } + profile = createSpecialProfile(id, path) + } else if specialProfileNeedsReset(profile) { + log.Debugf("profile: resetting special profile %s", id) + profile = createSpecialProfile(id, path) + } + if profile == nil { + return nil, errors.New("given ID is not a special profile ID") + } + + // Prepare profile for first use. + + // If we are refetching, assign the layered profile from the previous version. + // The internal references will be updated when the layered profile checks for updates. + if previousVersion != nil && previousVersion.layeredProfile != nil { + profile.layeredProfile = previousVersion.layeredProfile + } + + // Profiles must have a layered profile, create a new one if it + // does not yet exist. + if profile.layeredProfile == nil { + profile.layeredProfile = NewLayeredProfile(profile) + } + + // Add the profile to the currently active profiles. + addActiveProfile(profile) + + return profile, nil +} + func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (ok, changed bool) { // Get new profile name and check if profile is applicable to special handling. var newProfileName, newDescription string @@ -115,35 +185,45 @@ func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (ok, chan changed = true } - // Update LinkedPath to new value. - if profile.LinkedPath != binaryPath { - profile.LinkedPath = binaryPath + // Update PresentationPath to new value. + if profile.PresentationPath != binaryPath { + profile.PresentationPath = binaryPath changed = true } return true, changed } -func getSpecialProfile(profileID, linkedPath string) *Profile { +func createSpecialProfile(profileID string, path string) *Profile { switch profileID { case UnidentifiedProfileID: - return New(SourceLocal, UnidentifiedProfileID, linkedPath, nil) + return New(&Profile{ + ID: UnidentifiedProfileID, + Source: SourceLocal, + PresentationPath: path, + }) case SystemProfileID: - return New(SourceLocal, SystemProfileID, linkedPath, nil) + return New(&Profile{ + ID: SystemProfileID, + Source: SourceLocal, + PresentationPath: path, + }) case SystemResolverProfileID: - systemResolverProfile := New( - SourceLocal, - SystemResolverProfileID, - linkedPath, - map[string]interface{}{ + return New(&Profile{ + ID: SystemResolverProfileID, + Source: SourceLocal, + PresentationPath: path, + Config: map[string]interface{}{ // Explicitly setting the default action to "permit" will improve the // user experience for people who set the global default to "prompt". // Resolved domain from the system resolver are checked again when // attributed to a connection of a regular process. Otherwise, users // would see two connection prompts for the same domain. CfgOptionDefaultActionKey: "permit", + // Explicitly allow incoming connections. + CfgOptionBlockInboundKey: status.SecurityLevelOff, // Explicitly allow localhost and answers to multicast protocols that // are commonly used by system resolvers. // TODO: When the Portmaster gains the ability to attribute multicast @@ -154,6 +234,7 @@ func getSpecialProfile(profileID, linkedPath string) *Profile { "+ LAN UDP/5353", // Allow inbound mDNS requests and multicast replies. "+ LAN UDP/5355", // Allow inbound LLMNR requests and multicast replies. "+ LAN UDP/1900", // Allow inbound SSDP requests and multicast replies. + "- *", // Deny everything else. }, // Explicitly disable all filter lists, as these will be checked later // with the attributed connection. As this is the system resolver, this @@ -161,44 +242,44 @@ func getSpecialProfile(profileID, linkedPath string) *Profile { // the system resolver is used. Users who want to CfgOptionFilterListsKey: []string{}, }, - ) - return systemResolverProfile + }) case PortmasterProfileID: - profile := New(SourceLocal, PortmasterProfileID, linkedPath, nil) - profile.Internal = true - return profile + return New(&Profile{ + ID: PortmasterProfileID, + Source: SourceLocal, + PresentationPath: path, + Internal: true, + }) case PortmasterAppProfileID: - profile := New( - SourceLocal, - PortmasterAppProfileID, - linkedPath, - map[string]interface{}{ + return New(&Profile{ + ID: PortmasterAppProfileID, + Source: SourceLocal, + PresentationPath: path, + Config: map[string]interface{}{ CfgOptionDefaultActionKey: "block", CfgOptionEndpointsKey: []string{ "+ Localhost", "+ .safing.io", }, }, - ) - profile.Internal = true - return profile + Internal: true, + }) case PortmasterNotifierProfileID: - profile := New( - SourceLocal, - PortmasterNotifierProfileID, - linkedPath, - map[string]interface{}{ + return New(&Profile{ + ID: PortmasterNotifierProfileID, + Source: SourceLocal, + PresentationPath: path, + Config: map[string]interface{}{ CfgOptionDefaultActionKey: "block", CfgOptionEndpointsKey: []string{ "+ Localhost", }, }, - ) - profile.Internal = true - return profile + Internal: true, + }) default: return nil From d60329e1309a9183fd3261a55bcfb1d5b37f1f25 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 3 Oct 2022 22:11:40 +0200 Subject: [PATCH 02/15] Collect env, tags and and matching path for processes --- process/process.go | 69 +++++++++++++++++++++++++++++++-------- process/profile.go | 58 ++++++++++++++++++++++----------- process/tags.go | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 32 deletions(-) create mode 100644 process/tags.go diff --git a/process/process.go b/process/process.go index 652c1118..3bc261bf 100644 --- a/process/process.go +++ b/process/process.go @@ -42,14 +42,18 @@ type Process struct { Cwd string CmdLine string FirstArg string - - // SpecialDetail holds special information, the meaning of which can change - // based on any of the previous attributes. - SpecialDetail string + Env map[string]string // Profile attributes. // Once set, these don't change; safe for concurrent access. + // Tags holds extended information about the (virtual) process, which is used + // to find a profile. + Tags []profile.Tag + // MatchingPath holds an alternative binary path that can be used to find a + // profile. + MatchingPath string + // PrimaryProfileID holds the scoped ID of the primary profile. PrimaryProfileID string // profile holds the layered profile based on the primary profile. @@ -177,7 +181,7 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { } // Get process information from the system. - pInfo, err := processInfo.NewProcess(int32(pid)) + pInfo, err := processInfo.NewProcessWithContext(ctx, int32(pid)) if err != nil { return nil, err } @@ -186,7 +190,7 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { // net yet implemented for windows if onLinux { var uids []int32 - uids, err = pInfo.Uids() + uids, err = pInfo.UidsWithContext(ctx) if err != nil { return nil, fmt.Errorf("failed to get UID for p%d: %w", pid, err) } @@ -194,7 +198,7 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { } // Username - process.UserName, err = pInfo.Username() + process.UserName, err = pInfo.UsernameWithContext(ctx) if err != nil { return nil, fmt.Errorf("process: failed to get Username for p%d: %w", pid, err) } @@ -203,14 +207,14 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { // new.UserHome, err = // PPID - ppid, err := pInfo.Ppid() + ppid, err := pInfo.PpidWithContext(ctx) if err != nil { return nil, fmt.Errorf("failed to get PPID for p%d: %w", pid, err) } process.ParentPid = int(ppid) // Path - process.Path, err = pInfo.Exe() + process.Path, err = pInfo.ExeWithContext(ctx) if err != nil { return nil, fmt.Errorf("failed to get Path for p%d: %w", pid, err) } @@ -229,13 +233,13 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { // } // Command line arguments - process.CmdLine, err = pInfo.Cmdline() + process.CmdLine, err = pInfo.CmdlineWithContext(ctx) if err != nil { return nil, fmt.Errorf("failed to get Cmdline for p%d: %w", pid, err) } // Name - process.Name, err = pInfo.Name() + process.Name, err = pInfo.NameWithContext(ctx) if err != nil { return nil, fmt.Errorf("failed to get Name for p%d: %w", pid, err) } @@ -243,9 +247,48 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { process.Name = process.ExecName } - // OS specifics - process.specialOSInit() + // Get all environment variables + env, err := pInfo.EnvironWithContext(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get the environment for p%d: %w", pid, err) + } + // Split env variables in key and value. + process.Env = make(map[string]string, len(env)) + for _, entry := range env { + splitted := strings.SplitN(entry, "=", 2) + if len(splitted) == 2 { + process.Env[strings.Trim(splitted[0], `'"`)] = strings.Trim(splitted[1], `'"`) + } + } + + // Add process tags. + process.addTags() + if len(process.Tags) > 0 { + log.Tracer(ctx).Debugf("profile: added tags: %+v", process.Tags) + } process.Save() return process, nil } + +// MatchingData returns the matching data for the process. +func (p *Process) MatchingData() *MatchingData { + return &MatchingData{p} +} + +// MatchingData provides a interface compatible view on the process for profile matching. +type MatchingData struct { + p *Process +} + +// Tags returns process.Tags. +func (md *MatchingData) Tags() []profile.Tag { return md.p.Tags } + +// Env returns process.Env. +func (md *MatchingData) Env() map[string]string { return md.p.Env } + +// Path returns process.Path. +func (md *MatchingData) Path() string { return md.p.Path } + +// MatchingPath returns process.MatchingPath. +func (md *MatchingData) MatchingPath() string { return md.p.MatchingPath } diff --git a/process/profile.go b/process/profile.go index 6c8ceac6..aab86996 100644 --- a/process/profile.go +++ b/process/profile.go @@ -2,6 +2,7 @@ package process import ( "context" + "fmt" "os" "runtime" "strings" @@ -29,25 +30,48 @@ func (p *Process) GetProfile(ctx context.Context) (changed bool, err error) { // If not, continue with loading the profile. log.Tracer(ctx).Trace("process: loading profile") + // Check if there is a special profile for this process. + localProfile, err := p.loadSpecialProfile(ctx) + if err != nil { + return false, fmt.Errorf("failed to load special profile: %w", err) + } + + // Otherwise, find a regular profile for the process. + if localProfile == nil { + localProfile, err = profile.GetLocalProfile("", p.MatchingData(), p.CreateProfileCallback) + if err != nil { + return false, fmt.Errorf("failed to find profile: %w", err) + } + } + + // Assign profile to process. + p.PrimaryProfileID = localProfile.ScopedID() + p.profile = localProfile.LayeredProfile() + + return true, nil +} + +// loadSpecialProfile attempts to load a special profile. +func (p *Process) loadSpecialProfile(_ context.Context) (*profile.Profile, error) { // Check if we need a special profile. - profileID := "" + var specialProfileID string switch p.Pid { case UnidentifiedProcessID: - profileID = profile.UnidentifiedProfileID + specialProfileID = profile.UnidentifiedProfileID case UnsolicitedProcessID: - profileID = profile.UnsolicitedProfileID + specialProfileID = profile.UnsolicitedProfileID case SystemProcessID: - profileID = profile.SystemProfileID + specialProfileID = profile.SystemProfileID case ownPID: - profileID = profile.PortmasterProfileID + specialProfileID = profile.PortmasterProfileID default: // Check if this is another Portmaster component. if updatesPath != "" && strings.HasPrefix(p.Path, updatesPath) { switch { case strings.Contains(p.Path, "portmaster-app"): - profileID = profile.PortmasterAppProfileID + specialProfileID = profile.PortmasterAppProfileID case strings.Contains(p.Path, "portmaster-notifier"): - profileID = profile.PortmasterNotifierProfileID + specialProfileID = profile.PortmasterNotifierProfileID default: // Unexpected binary from within the Portmaster updates directpry. log.Warningf("process: unexpected binary in the updates directory: %s", p.Path) @@ -62,10 +86,10 @@ func (p *Process) GetProfile(ctx context.Context) (changed bool, err error) { if (p.Path == `C:\Windows\System32\svchost.exe` || p.Path == `C:\Windows\system32\svchost.exe`) && // This comes from the windows tasklist command and should be pretty consistent. - (strings.Contains(p.SpecialDetail, "Dnscache") || + (profile.KeyAndValueInTags(p.Tags, "svchost", "Dnscache") || // As an alternative in case of failure, we try to match the svchost.exe service parameter. strings.Contains(p.CmdLine, "-s Dnscache")) { - profileID = profile.SystemResolverProfileID + specialProfileID = profile.SystemResolverProfileID } case "linux": switch p.Path { @@ -77,22 +101,18 @@ func (p *Process) GetProfile(ctx context.Context) (changed bool, err error) { "/usr/sbin/nscd", "/usr/bin/dnsmasq", "/usr/sbin/dnsmasq": - profileID = profile.SystemResolverProfileID + specialProfileID = profile.SystemResolverProfileID } } } - // Get the (linked) local profile. - localProfile, err := profile.GetProfile(profile.SourceLocal, profileID, p.Path, false) - if err != nil { - return false, err + // Check if a special profile should be applied. + if specialProfileID == "" { + return nil, nil } - // Assign profile to process. - p.PrimaryProfileID = localProfile.ScopedID() - p.profile = localProfile.LayeredProfile() - - return true, nil + // Return special profile. + return profile.GetSpecialProfile(specialProfileID, p.Path) } // UpdateProfileMetadata updates the metadata of the local profile diff --git a/process/tags.go b/process/tags.go new file mode 100644 index 00000000..0eea7f49 --- /dev/null +++ b/process/tags.go @@ -0,0 +1,80 @@ +package process + +import ( + "errors" + "sync" + + "github.com/safing/portmaster/profile" +) + +var ( + tagRegistry []TagHandler + tagRegistryLock sync.RWMutex +) + +// TagHandler is a collection of process tag related interfaces. +type TagHandler interface { + // Name returns the tag handler name. + Name() string + + // TagDescriptions returns a list of all possible tags and their description + // of this handler. + TagDescriptions() []TagDescription + + // AddTags adds tags to the given process. + AddTags(p *Process) + + // CreateProfile creates a profile based on the tags of the process. + // Returns nil to skip. + CreateProfile(p *Process) *profile.Profile +} + +// TagDescription describes a tag. +type TagDescription struct { + ID string + Name string + Description string +} + +// RegisterTagHandler registers a tag handler. +func RegisterTagHandler(th TagHandler) error { + tagRegistryLock.Lock() + defer tagRegistryLock.Unlock() + + // Check if the handler is already registered. + for _, existingTH := range tagRegistry { + if th.Name() == existingTH.Name() { + return errors.New("already registered") + } + } + + tagRegistry = append(tagRegistry, th) + return nil +} + +func (p *Process) addTags() { + tagRegistryLock.RLock() + defer tagRegistryLock.RUnlock() + + for _, th := range tagRegistry { + th.AddTags(p) + } +} + +// CreateProfileCallback attempts to create a profile on special attributes +// of the process. +func (p *Process) CreateProfileCallback() *profile.Profile { + tagRegistryLock.RLock() + defer tagRegistryLock.RUnlock() + + // Go through handlers and see which one wants to create a profile. + for _, th := range tagRegistry { + newProfile := th.CreateProfile(p) + if newProfile != nil { + return newProfile + } + } + + // No handler wanted to create a profile. + return nil +} From fd0ce5732d8bad5c8e06484885974bfe046c55e8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 3 Oct 2022 22:12:04 +0200 Subject: [PATCH 03/15] Add first tag handlers for svchost, app image, net --- network/connection.go | 1 + process/find.go | 9 ++- process/process_default.go | 8 +-- process/process_linux.go | 3 - process/process_windows.go | 24 -------- process/tags/appimage_unix.go | 87 +++++++++++++++++++++++++++++ process/tags/net.go | 65 ++++++++++++++++++++++ process/tags/svchost_windows.go | 97 +++++++++++++++++++++++++++++++++ 8 files changed, 259 insertions(+), 35 deletions(-) create mode 100644 process/tags/appimage_unix.go create mode 100644 process/tags/net.go create mode 100644 process/tags/svchost_windows.go diff --git a/network/connection.go b/network/connection.go index e61d12fa..d87964f9 100644 --- a/network/connection.go +++ b/network/connection.go @@ -14,6 +14,7 @@ import ( "github.com/safing/portmaster/network/netutils" "github.com/safing/portmaster/network/packet" "github.com/safing/portmaster/process" + _ "github.com/safing/portmaster/process/tags" "github.com/safing/portmaster/resolver" "github.com/safing/spn/navigator" ) diff --git a/process/find.go b/process/find.go index 9c395f14..9f021426 100644 --- a/process/find.go +++ b/process/find.go @@ -69,13 +69,18 @@ func GetNetworkHost(ctx context.Context, remoteIP net.IP) (process *Process, err UserID: NetworkHostProcessID, Pid: NetworkHostProcessID, ParentPid: NetworkHostProcessID, - Path: fmt.Sprintf("net:%s", remoteIP), + Tags: []profile.Tag{ + { + Key: "net", + Value: remoteIP.String(), + }, + }, FirstSeen: now, LastSeen: now, } // Get the (linked) local profile. - networkHostProfile, err := profile.GetProfile(profile.SourceNetwork, remoteIP.String(), "", false) + networkHostProfile, err := profile.GetLocalProfile("", networkHost.MatchingData(), networkHost.CreateProfileCallback) if err != nil { return nil, err } diff --git a/process/process_default.go b/process/process_default.go index 97f093e9..4d9d58dd 100644 --- a/process/process_default.go +++ b/process/process_default.go @@ -1,11 +1,7 @@ -//+build !windows,!linux +//go:build !windows && !linux +// +build !windows,!linux package process // SystemProcessID is the PID of the System/Kernel itself. const SystemProcessID = 0 - -// specialOSInit does special OS specific Process initialization. -func (p *Process) specialOSInit() { - -} diff --git a/process/process_linux.go b/process/process_linux.go index 64268a66..3d62bb9b 100644 --- a/process/process_linux.go +++ b/process/process_linux.go @@ -2,6 +2,3 @@ package process // SystemProcessID is the PID of the System/Kernel itself. const SystemProcessID = 0 - -// specialOSInit does special OS specific Process initialization. -func (p *Process) specialOSInit() {} diff --git a/process/process_windows.go b/process/process_windows.go index c0f722c2..e350f5a7 100644 --- a/process/process_windows.go +++ b/process/process_windows.go @@ -1,28 +1,4 @@ package process -import ( - "fmt" - - "github.com/safing/portbase/log" - "github.com/safing/portbase/utils/osdetail" -) - // SystemProcessID is the PID of the System/Kernel itself. const SystemProcessID = 4 - -// specialOSInit does special OS specific Process initialization. -func (p *Process) specialOSInit() { - // add svchost.exe service names to Name - if p.ExecName == "svchost.exe" { - svcNames, err := osdetail.GetServiceNames(int32(p.Pid)) - switch err { - case nil: - p.Name += fmt.Sprintf(" (%s)", svcNames) - p.SpecialDetail = svcNames - case osdetail.ErrServiceNotFound: - log.Tracef("process: failed to get service name for svchost.exe (pid %d): %s", p.Pid, err) - default: - log.Warningf("process: failed to get service name for svchost.exe (pid %d): %s", p.Pid, err) - } - } -} diff --git a/process/tags/appimage_unix.go b/process/tags/appimage_unix.go new file mode 100644 index 00000000..54e9d8bb --- /dev/null +++ b/process/tags/appimage_unix.go @@ -0,0 +1,87 @@ +package tags + +import ( + "strings" + + "github.com/safing/portmaster/process" + "github.com/safing/portmaster/profile" +) + +func init() { + err := process.RegisterTagHandler(new(AppImageHandler)) + if err != nil { + panic(err) + } +} + +const ( + appImageName = "AppImage" + appImagePathTagKey = "app-image-path" +) + +// AppImageHandler handles AppImage processes on Unix systems. +type AppImageHandler struct{} + +// Name returns the tag handler name. +func (h *AppImageHandler) Name() string { + return appImageName +} + +// TagDescriptions returns a list of all possible tags and their description +// of this handler. +func (h *AppImageHandler) TagDescriptions() []process.TagDescription { + return []process.TagDescription{ + { + ID: appImagePathTagKey, + Name: "App Image Path", + Description: "Path to the app image file itself.", + }, + } +} + +// AddTags adds tags to the given process. +func (h *AppImageHandler) AddTags(p *process.Process) { + // Get and verify AppImage location. + appImageLocation, ok := p.Env["APPIMAGE"] + if !ok { + return + } + appImageMountDir, ok := p.Env["APPDIR"] + if !ok { + return + } + // Check if the process path is in the mount dir. + if !strings.HasPrefix(p.Path, appImageMountDir) { + return + } + + // Add matching path for regular profile matching. + p.MatchingPath = appImageLocation + + // Add app image tags. + p.Tags = append(p.Tags, profile.Tag{ + Key: appImagePathTagKey, + Value: appImageLocation, + }) +} + +// CreateProfile creates a profile based on the tags of the process. +// Returns nil to skip. +func (h *AppImageHandler) CreateProfile(p *process.Process) *profile.Profile { + for _, tag := range p.Tags { + if tag.Key == appImagePathTagKey { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + PresentationPath: p.Path, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypePathID, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, // Value of appImagePathTagKey. + }, + }, + }) + } + } + return nil +} diff --git a/process/tags/net.go b/process/tags/net.go new file mode 100644 index 00000000..db2848b0 --- /dev/null +++ b/process/tags/net.go @@ -0,0 +1,65 @@ +package tags + +import ( + "github.com/safing/portmaster/process" + "github.com/safing/portmaster/profile" +) + +func init() { + err := process.RegisterTagHandler(new(NetworkHandler)) + if err != nil { + panic(err) + } +} + +const ( + netName = "Network" + netTagKey = "net" +) + +// NetworkHandler handles AppImage processes on Unix systems. +type NetworkHandler struct{} + +// Name returns the tag handler name. +func (h *NetworkHandler) Name() string { + return netName +} + +// TagDescriptions returns a list of all possible tags and their description +// of this handler. +func (h *NetworkHandler) TagDescriptions() []process.TagDescription { + return []process.TagDescription{ + { + ID: netTagKey, + Name: "Things on the Network", + Description: "Other (virtual) devices that make requests to Portmaster, if enabled in Portmaster.", + }, + } +} + +// AddTags adds tags to the given process. +func (h *NetworkHandler) AddTags(p *process.Process) { + // The "net" tag is added directly when creating the virtual process. +} + +// CreateProfile creates a profile based on the tags of the process. +// Returns nil to skip. +func (h *NetworkHandler) CreateProfile(p *process.Process) *profile.Profile { + for _, tag := range p.Tags { + if tag.Key == netTagKey { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: p.Name, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, + }, + }, + }) + } + } + return nil +} diff --git a/process/tags/svchost_windows.go b/process/tags/svchost_windows.go new file mode 100644 index 00000000..724c29a8 --- /dev/null +++ b/process/tags/svchost_windows.go @@ -0,0 +1,97 @@ +package tags + +import ( + "fmt" + "strings" + + "github.com/safing/portbase/log" + "github.com/safing/portmaster/process" + "github.com/safing/portmaster/profile" +) + +func init() { + err := process.RegisterTagHandler(new(SVCHostTagHandler)) + if err != nil { + panic(err) + } +} + +const ( + svchostName = "SvcHost" + svchostTagKey = "svchost" +) + +// SVCHostTagHandler handles svchost processes on Windows. +type SVCHostTagHandler struct{} + +// Name returns the tag handler name. +func (h *SVCHostTagHandler) Name() string { + return svcHostName +} + +// TagDescriptions returns a list of all possible tags and their description +// of this handler. +func (h *SVCHostTagHandler) TagDescriptions() []process.TagDescription { + return []process.TagDescription{ + process.TagDescription{ + ID: svcHostTagKey, + Name: "SvcHost Service Name", + Description: "Name of a service running in svchost.exe as reported by Windows.", + }, + } +} + +// TagKeys returns a list of all possible tag keys of this handler. +func (h *SVCHostTagHandler) TagKeys() []string { + return []string{svcHostTagKey} +} + +// AddTags adds tags to the given process. +func (h *SVCHostTagHandler) AddTags(p *process.Process) { + // Check for svchost.exe. + if p.ExecName != "svchost.exe" { + return + } + + // Get services of svchost instance. + svcNames, err := osdetail.GetServiceNames(int32(p.Pid)) + switch err { + case nil: + // Append service names to process name. + p.Name += fmt.Sprintf(" (%s)", strings.Join(svcNames, ", ")) + // Add services as tags. + for _, svcName := range svcNames { + p.Tag = append(p.Tag, profile.Tag{ + Key: svchostTagKey, + Value: svcName, + }) + } + case osdetail.ErrServiceNotFound: + log.Tracef("process/tags: failed to get service name for svchost.exe (pid %d): %s", p.Pid, err) + default: + log.Warningf("process/tags: failed to get service name for svchost.exe (pid %d): %s", p.Pid, err) + } +} + +// CreateProfile creates a profile based on the tags of the process. +// Returns nil to skip. +func (h *SVCHostTagHandler) CreateProfile(p *process.Process) *profile.Profile { + for _, tag := range p.Tags { + if tag.Key == svchostTagKey { + return profile.New( + profile.SourceLocal, + "", + "Windows Service: "+tag.Value, + p.Path, + []profile.Fingerprint{profile.Fingerprint{ + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, + }}, + nil, + ) + } + } + return nil +} From f4490b4f11bd4875c91d6f133fd8f4a66170df92 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 3 Oct 2022 22:12:58 +0200 Subject: [PATCH 04/15] Use new profile functions in firewall --- firewall/master.go | 5 ++++- firewall/prompt.go | 10 ++++++---- firewall/tunnel.go | 5 ++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index 1bad4fb2..bd7a9c7d 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -71,7 +71,10 @@ func DecideOnConnection(ctx context.Context, conn *network.Connection, pkt packe // Check if the layered profile needs updating. if layeredProfile.NeedsUpdate() { // Update revision counter in connection. - conn.ProfileRevisionCounter = layeredProfile.Update() + conn.ProfileRevisionCounter = layeredProfile.Update( + conn.Process().MatchingData(), + conn.Process().CreateProfileCallback, + ) conn.SaveWhenFinished() // Reset verdict for connection. diff --git a/firewall/prompt.go b/firewall/prompt.go index 0356516a..18ef160b 100644 --- a/firewall/prompt.go +++ b/firewall/prompt.go @@ -177,9 +177,11 @@ func createPrompt(ctx context.Context, conn *network.Connection, pkt packet.Pack EventData: &promptData{ Entity: entity, Profile: promptProfile{ - Source: string(localProfile.Source), - ID: localProfile.ID, - LinkedPath: localProfile.LinkedPath, + Source: string(localProfile.Source), + ID: localProfile.ID, + // LinkedPath is used to enhance the display of the prompt in the UI. + // TODO: Using the process path is a workaround. Find a cleaner solution. + LinkedPath: conn.Process().Path, }, }, Expires: expires, @@ -259,7 +261,7 @@ func saveResponse(p *profile.Profile, entity *intel.Entity, promptResponse strin // Update the profile if necessary. if p.IsOutdated() { var err error - p, err = profile.GetProfile(p.Source, p.ID, p.LinkedPath, false) + p, err = profile.GetLocalProfile(p.ID, nil, nil) if err != nil { return err } diff --git a/firewall/tunnel.go b/firewall/tunnel.go index 232636f6..e062512c 100644 --- a/firewall/tunnel.go +++ b/firewall/tunnel.go @@ -70,7 +70,10 @@ func checkTunneling(ctx context.Context, conn *network.Connection) { // Update profile. if layeredProfile.NeedsUpdate() { // Update revision counter in connection. - conn.ProfileRevisionCounter = layeredProfile.Update() + conn.ProfileRevisionCounter = layeredProfile.Update( + conn.Process().MatchingData(), + conn.Process().CreateProfileCallback, + ) conn.SaveWhenFinished() } else { // Check if the revision counter of the connection needs updating. From c22fb3316b33d5a2d00ce054d88651102c75c46c Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 4 Oct 2022 12:52:50 +0200 Subject: [PATCH 05/15] Add support for Windows Store tags --- process/tags/winstore_windows.go | 120 +++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 process/tags/winstore_windows.go diff --git a/process/tags/winstore_windows.go b/process/tags/winstore_windows.go new file mode 100644 index 00000000..86cf1563 --- /dev/null +++ b/process/tags/winstore_windows.go @@ -0,0 +1,120 @@ +package tags + +import ( + "os" + "strings" + + "github.com/safing/portbase/log" + + "github.com/safing/portbase/utils" + + "github.com/safing/portmaster/process" + "github.com/safing/portmaster/profile" +) + +func init() { + err := process.RegisterTagHandler(new(WinStoreHandler)) + if err != nil { + panic(err) + } + + // Add custom WindowsApps path. + customWinStorePath := os.ExpandEnv(`%ProgramFiles%\WindowsApps\`) + if !utils.StringSliceEqual(winStorePaths, customWinStorePath) { + winStorePaths = append(winStorePaths, customWinStorePath) + } +} + +const ( + winStoreName = "Windows Store" + winStoreAppNameTagKey = "winstore-app-name" + winStorePublisherIDTagKey = "winstore-publisher-id" +) + +var winStorePaths = []string{`C:\Program Files\WindowsApps\`} + +// WinStoreHandler handles AppImage processes on Unix systems. +type WinStoreHandler struct{} + +// Name returns the tag handler name. +func (h *WinStoreHandler) Name() string { + return winStoreName +} + +// TagDescriptions returns a list of all possible tags and their description +// of this handler. +func (h *WinStoreHandler) TagDescriptions() []process.TagDescription { + return []process.TagDescription{ + { + ID: winStoreAppNameTagKey, + Name: "Windows Store App Name", + Description: "Name of the Windows Store App, as found in the executable path.", + }, + { + ID: winStorePublisherIDTagKey, + Name: "Windows Store Publisher ID", + Description: "Publisher ID of a Windows Store App.", + }, + } +} + +// AddTags adds tags to the given process. +func (h *WinStoreHandler) AddTags(p *process.Process) { + // Check if the path is in one of the Windows Store Apps paths. + var appDir string + for _, winStorePath := range winStorePaths { + if strings.HasPrefix(p.Path, winStorePath) { + appDir := strings.SplitN(strings.TrimPrefix(p.Path, winStorePath), `\`, 2)[0] + break + } + } + if appDir == "" { + return + } + + // Extract information from path. + // Example: Microsoft.Office.OneNote_17.6769.57631.0_x64__8wekyb3d8bbwe + splitted := strings.Split(appDir, "_") + if splitted != 5 { // Four fields, one "__". + log.Debugf("profile/tags: windows store app has incompatible app dir format: %q", appDir) + return + } + + name := splitted[0] + // version := splitted[1] + // platform := splitted[2] + publisherID := splitted[4] + + // Add tags. + p.Tags = append(p.Tags, profile.Tag{ + Key: winStoreAppNameTagKey, + Value: name, + }) + p.Tags = append(p.Tags, profile.Tag{ + Key: winStorePublisherIDTagKey, + Value: publisherID, + }) +} + +// CreateProfile creates a profile based on the tags of the process. +// Returns nil to skip. +func (h *WinStoreHandler) CreateProfile(p *process.Process) *profile.Profile { + for _, tag := range p.Tags { + if tag.Key == winStoreAppNameTagKey { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: tag.Value, + PresentationPath: p.Path, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, // Value of appImagePathTagKey. + }, + }, + }) + } + } + return nil +} From f329e40da79ab0502725cef6e82314ca4a10a4e5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 4 Oct 2022 12:53:04 +0200 Subject: [PATCH 06/15] Improve network tags for external requests --- process/tags/net.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/process/tags/net.go b/process/tags/net.go index db2848b0..8c6196e5 100644 --- a/process/tags/net.go +++ b/process/tags/net.go @@ -13,8 +13,8 @@ func init() { } const ( - netName = "Network" - netTagKey = "net" + netName = "Network" + netIPTagKey = "ip" ) // NetworkHandler handles AppImage processes on Unix systems. @@ -30,9 +30,9 @@ func (h *NetworkHandler) Name() string { func (h *NetworkHandler) TagDescriptions() []process.TagDescription { return []process.TagDescription{ { - ID: netTagKey, - Name: "Things on the Network", - Description: "Other (virtual) devices that make requests to Portmaster, if enabled in Portmaster.", + ID: netIPTagKey, + Name: "IP Address", + Description: "The remote IP address of external requests to Portmaster, if enabled.", }, } } @@ -46,7 +46,7 @@ func (h *NetworkHandler) AddTags(p *process.Process) { // Returns nil to skip. func (h *NetworkHandler) CreateProfile(p *process.Process) *profile.Profile { for _, tag := range p.Tags { - if tag.Key == netTagKey { + if tag.Key == netIPTagKey { return profile.New(&profile.Profile{ Source: profile.SourceLocal, Name: p.Name, From fcf603ea90d25fab5a0dd73ac96892b89590ea89 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 4 Oct 2022 12:53:25 +0200 Subject: [PATCH 07/15] Check all regex fingerprint matches --- profile/fingerprint.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 1fb44fbf..4b71cddf 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -22,7 +22,7 @@ import ( // 3. How "strong" was the match? // 1. Equals: Length of path (irrelevant) // 2. Prefix: Length of prefix -// 3. Regex: 0 (we are not suicidal) +// 3. Regex: Length of match // ms-store:Microsoft.One.Note @@ -137,12 +137,20 @@ type fingerprintRegex struct { } func (fp fingerprintRegex) Match(value string) (score int) { - if fp.regex.MatchString(value) { - // Do not return any deviation from the base score. - // Trying to assign different scores to regex probably won't turn out to - // be a good idea. - return fingerprintRegexBaseScore + // Find best match. + for _, match := range fp.regex.FindAllString(value, -1) { + // Save match length if higher than score. + // This will also ignore empty matches. + if len(match) > score { + score = len(match) + } } + + // Add base score and return if anything was found. + if score > 0 { + return fingerprintRegexBaseScore + checkMatchStrength(score) + } + return 0 } From c4943a96b1ae3995c4c83002f68380f60cf389b5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 4 Oct 2022 12:53:42 +0200 Subject: [PATCH 08/15] Expose registered process tags via API --- process/api.go | 39 +++++++++++++++++++++++++++++++++++++++ process/module.go | 4 ++++ 2 files changed, 43 insertions(+) create mode 100644 process/api.go diff --git a/process/api.go b/process/api.go new file mode 100644 index 00000000..2e37502d --- /dev/null +++ b/process/api.go @@ -0,0 +1,39 @@ +package process + +import ( + "github.com/safing/portbase/api" +) + +func registerAPIEndpoints() error { + if err := api.RegisterEndpoint(api.Endpoint{ + Path: "process/tags", + Read: api.PermitUser, + BelongsTo: module, + StructFunc: handleProcessTagMetadata, + Name: "Get Process Tag Metadata", + Description: "Get information about process tags.", + }); err != nil { + return err + } + + return nil +} + +func handleProcessTagMetadata(ar *api.Request) (i interface{}, err error) { + tagRegistryLock.Lock() + defer tagRegistryLock.Unlock() + + // Create response struct. + resp := struct { + Tags []TagDescription + }{ + Tags: make([]TagDescription, 0, len(tagRegistry)*2), + } + + // Get all tag descriptions. + for _, th := range tagRegistry { + resp.Tags = append(resp.Tags, th.TagDescriptions()...) + } + + return resp, nil +} diff --git a/process/module.go b/process/module.go index a530bd21..b33be8ca 100644 --- a/process/module.go +++ b/process/module.go @@ -26,5 +26,9 @@ func start() error { updatesPath += string(os.PathSeparator) } + if err := registerAPIEndpoints(); err != nil { + return err + } + return nil } From 595f4c0106cf283dc0d5ab1d3ceb70a9c1044904 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 6 Oct 2022 10:33:25 +0200 Subject: [PATCH 09/15] Improve profile metadata handling --- process/find.go | 16 +---- process/profile.go | 24 ------- process/tags/appimage_unix.go | 7 +- process/tags/svchost_windows.go | 37 +++++----- process/tags/winstore_windows.go | 15 ++-- profile/fingerprint.go | 5 +- profile/get.go | 34 +++++++-- profile/profile.go | 117 ++++++++++++++++--------------- profile/special.go | 47 ++++++++++--- 9 files changed, 165 insertions(+), 137 deletions(-) diff --git a/process/find.go b/process/find.go index 9f021426..35ff4b75 100644 --- a/process/find.go +++ b/process/find.go @@ -64,14 +64,14 @@ func GetProcessByConnection(ctx context.Context, pktInfo *packet.Info) (process func GetNetworkHost(ctx context.Context, remoteIP net.IP) (process *Process, err error) { //nolint:interfacer now := time.Now().Unix() networkHost := &Process{ - Name: fmt.Sprintf("Network Host %s", remoteIP), - UserName: "Unknown", + Name: fmt.Sprintf("Device at %s", remoteIP), + UserName: "N/A", UserID: NetworkHostProcessID, Pid: NetworkHostProcessID, ParentPid: NetworkHostProcessID, Tags: []profile.Tag{ { - Key: "net", + Key: "ip", Value: remoteIP.String(), }, }, @@ -89,16 +89,6 @@ func GetNetworkHost(ctx context.Context, remoteIP net.IP) (process *Process, err networkHost.PrimaryProfileID = networkHostProfile.ScopedID() networkHost.profile = networkHostProfile.LayeredProfile() - if networkHostProfile.Name == "" { - // Assign name and save. - networkHostProfile.Name = networkHost.Name - - err := networkHostProfile.Save() - if err != nil { - log.Warningf("process: failed to save profile %s: %s", networkHostProfile.ScopedID(), err) - } - } - return networkHost, nil } diff --git a/process/profile.go b/process/profile.go index aab86996..9c49e832 100644 --- a/process/profile.go +++ b/process/profile.go @@ -15,9 +15,6 @@ var ownPID = os.Getpid() // GetProfile finds and assigns a profile set to the process. func (p *Process) GetProfile(ctx context.Context) (changed bool, err error) { - // Update profile metadata outside of *Process lock. - defer p.UpdateProfileMetadata() - p.Lock() defer p.Unlock() @@ -114,24 +111,3 @@ func (p *Process) loadSpecialProfile(_ context.Context) (*profile.Profile, error // Return special profile. return profile.GetSpecialProfile(specialProfileID, p.Path) } - -// UpdateProfileMetadata updates the metadata of the local profile -// as required. -func (p *Process) UpdateProfileMetadata() { - // Check if there is a profile to work with. - localProfile := p.Profile().LocalProfile() - if localProfile == nil { - return - } - - // Update metadata of profile. - metadataUpdated := localProfile.UpdateMetadata(p.Path) - - // Save the profile if we changed something. - if metadataUpdated { - err := localProfile.Save() - if err != nil { - log.Warningf("process: failed to save profile %s: %s", localProfile.ScopedID(), err) - } - } -} diff --git a/process/tags/appimage_unix.go b/process/tags/appimage_unix.go index 54e9d8bb..094a5f39 100644 --- a/process/tags/appimage_unix.go +++ b/process/tags/appimage_unix.go @@ -3,6 +3,7 @@ package tags import ( "strings" + "github.com/safing/portbase/utils/osdetail" "github.com/safing/portmaster/process" "github.com/safing/portmaster/profile" ) @@ -71,8 +72,10 @@ func (h *AppImageHandler) CreateProfile(p *process.Process) *profile.Profile { for _, tag := range p.Tags { if tag.Key == appImagePathTagKey { return profile.New(&profile.Profile{ - Source: profile.SourceLocal, - PresentationPath: p.Path, + Source: profile.SourceLocal, + Name: osdetail.GenerateBinaryNameFromPath(tag.Value), + PresentationPath: p.Path, + UsePresentationPath: true, Fingerprints: []profile.Fingerprint{ { Type: profile.FingerprintTypePathID, diff --git a/process/tags/svchost_windows.go b/process/tags/svchost_windows.go index 724c29a8..60ab951b 100644 --- a/process/tags/svchost_windows.go +++ b/process/tags/svchost_windows.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/safing/portbase/log" + "github.com/safing/portbase/utils/osdetail" "github.com/safing/portmaster/process" "github.com/safing/portmaster/profile" ) @@ -17,7 +18,7 @@ func init() { } const ( - svchostName = "SvcHost" + svchostName = "Service Host" svchostTagKey = "svchost" ) @@ -26,7 +27,7 @@ type SVCHostTagHandler struct{} // Name returns the tag handler name. func (h *SVCHostTagHandler) Name() string { - return svcHostName + return svchostName } // TagDescriptions returns a list of all possible tags and their description @@ -34,7 +35,7 @@ func (h *SVCHostTagHandler) Name() string { func (h *SVCHostTagHandler) TagDescriptions() []process.TagDescription { return []process.TagDescription{ process.TagDescription{ - ID: svcHostTagKey, + ID: svchostTagKey, Name: "SvcHost Service Name", Description: "Name of a service running in svchost.exe as reported by Windows.", }, @@ -43,7 +44,7 @@ func (h *SVCHostTagHandler) TagDescriptions() []process.TagDescription { // TagKeys returns a list of all possible tag keys of this handler. func (h *SVCHostTagHandler) TagKeys() []string { - return []string{svcHostTagKey} + return []string{svchostTagKey} } // AddTags adds tags to the given process. @@ -61,7 +62,7 @@ func (h *SVCHostTagHandler) AddTags(p *process.Process) { p.Name += fmt.Sprintf(" (%s)", strings.Join(svcNames, ", ")) // Add services as tags. for _, svcName := range svcNames { - p.Tag = append(p.Tag, profile.Tag{ + p.Tags = append(p.Tags, profile.Tag{ Key: svchostTagKey, Value: svcName, }) @@ -78,19 +79,19 @@ func (h *SVCHostTagHandler) AddTags(p *process.Process) { func (h *SVCHostTagHandler) CreateProfile(p *process.Process) *profile.Profile { for _, tag := range p.Tags { if tag.Key == svchostTagKey { - return profile.New( - profile.SourceLocal, - "", - "Windows Service: "+tag.Value, - p.Path, - []profile.Fingerprint{profile.Fingerprint{ - Type: profile.FingerprintTypeTagID, - Key: tag.Key, - Operation: profile.FingerprintOperationEqualsID, - Value: tag.Value, - }}, - nil, - ) + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: "Windows Service: " + osdetail.GenerateBinaryNameFromPath(tag.Value), + UsePresentationPath: false, + Fingerprints: []profile.Fingerprint{ + profile.Fingerprint{ + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, + }, + }, + }) } } return nil diff --git a/process/tags/winstore_windows.go b/process/tags/winstore_windows.go index 86cf1563..bc3b538d 100644 --- a/process/tags/winstore_windows.go +++ b/process/tags/winstore_windows.go @@ -4,6 +4,8 @@ import ( "os" "strings" + "github.com/safing/portbase/utils/osdetail" + "github.com/safing/portbase/log" "github.com/safing/portbase/utils" @@ -20,7 +22,7 @@ func init() { // Add custom WindowsApps path. customWinStorePath := os.ExpandEnv(`%ProgramFiles%\WindowsApps\`) - if !utils.StringSliceEqual(winStorePaths, customWinStorePath) { + if !utils.StringInSlice(winStorePaths, customWinStorePath) { winStorePaths = append(winStorePaths, customWinStorePath) } } @@ -64,7 +66,7 @@ func (h *WinStoreHandler) AddTags(p *process.Process) { var appDir string for _, winStorePath := range winStorePaths { if strings.HasPrefix(p.Path, winStorePath) { - appDir := strings.SplitN(strings.TrimPrefix(p.Path, winStorePath), `\`, 2)[0] + appDir = strings.SplitN(strings.TrimPrefix(p.Path, winStorePath), `\`, 2)[0] break } } @@ -75,7 +77,7 @@ func (h *WinStoreHandler) AddTags(p *process.Process) { // Extract information from path. // Example: Microsoft.Office.OneNote_17.6769.57631.0_x64__8wekyb3d8bbwe splitted := strings.Split(appDir, "_") - if splitted != 5 { // Four fields, one "__". + if len(splitted) != 5 { // Four fields, one "__". log.Debugf("profile/tags: windows store app has incompatible app dir format: %q", appDir) return } @@ -102,9 +104,10 @@ func (h *WinStoreHandler) CreateProfile(p *process.Process) *profile.Profile { for _, tag := range p.Tags { if tag.Key == winStoreAppNameTagKey { return profile.New(&profile.Profile{ - Source: profile.SourceLocal, - Name: tag.Value, - PresentationPath: p.Path, + Source: profile.SourceLocal, + Name: osdetail.GenerateBinaryNameFromPath(tag.Value), + PresentationPath: p.Path, + UsePresentationPath: true, Fingerprints: []profile.Fingerprint{ { Type: profile.FingerprintTypeTagID, diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 4b71cddf..b0177308 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -163,8 +163,9 @@ type parsedFingerprints struct { func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { parsed = &parsedFingerprints{} - // Add deprecated linked path to fingerprints. - if deprecatedLinkedPath != "" { + // Add deprecated LinkedPath to fingerprints, if they are empty. + // TODO: Remove in v1.5 + if len(raw) == 0 && deprecatedLinkedPath != "" { parsed.pathPrints = append(parsed.pathPrints, &fingerprintEquals{ Fingerprint: Fingerprint{ Type: FingerprintTypePathID, diff --git a/profile/get.go b/profile/get.go index d9820c18..ad8d018b 100644 --- a/profile/get.go +++ b/profile/get.go @@ -70,7 +70,10 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P } // If we still don't have a profile, create a new one. + var created bool if profile == nil { + created = true + // Try the profile creation callback, if we have one. if createProfileCallback != nil { profile = createProfileCallback() @@ -84,9 +87,10 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P } profile = New(&Profile{ - ID: id, - Source: SourceLocal, - PresentationPath: md.Path(), + ID: id, + Source: SourceLocal, + PresentationPath: md.Path(), + UsePresentationPath: true, Fingerprints: []Fingerprint{ { Type: FingerprintTypePathID, @@ -98,6 +102,25 @@ func GetLocalProfile(id string, md MatchingData, createProfileCallback func() *P } } + // Initialize and update profile. + + // Update metadata. + changed := profile.updateMetadata(md.Path()) + + // Save if created or changed. + if created || changed { + // Save profile. + err := profile.Save() + if err != nil { + log.Warningf("profile: failed to save profile %s after creation: %s", profile.ScopedID(), err) + } + } + + // Trigger further metadata fetching from system if profile was created. + if created && profile.UsePresentationPath { + module.StartWorker("get profile metadata", profile.updateMetadataFromSystem) + } + // Prepare profile for first use. // Process profiles are coming directly from the database or are new. @@ -158,7 +181,10 @@ profileFeed: prints, err := loadProfileFingerprints(r) if err != nil { log.Debugf("profile: failed to load fingerprints of %s: %s", r.Key(), err) - continue + } + // Continue with any returned fingerprints. + if prints == nil { + continue profileFeed } // Get matching score and compare. diff --git a/profile/profile.go b/profile/profile.go index 09dc8d68..fb56d90c 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "path/filepath" "strings" "sync" "sync/atomic" @@ -89,6 +88,10 @@ type Profile struct { //nolint:maligned // not worth the effort // Is automatically removed when the path does not exist. // Is automatically populated with the next match when empty. PresentationPath string + // UsePresentationPath can be used to enable/disable fetching information + // from the executable at PresentationPath. In some cases, this is not + // desirable. + UsePresentationPath bool // Fingerprints holds process matching information. Fingerprints []Fingerprint // SecurityLevel is the mininum security level to apply to @@ -419,48 +422,54 @@ func EnsureProfile(r record.Record) (*Profile, error) { return newProfile, nil } -// UpdateMetadata updates meta data fields on the profile and returns whether -// the profile was changed. If there is data that needs to be fetched from the -// operating system, it will start an async worker to fetch that data and save -// the profile afterwards. -func (profile *Profile) UpdateMetadata(binaryPath string) (changed bool) { +// updateMetadata updates meta data fields on the profile and returns whether +// the profile was changed. +func (profile *Profile) updateMetadata(binaryPath string) (changed bool) { // Check if this is a local profile, else warn and return. if profile.Source != SourceLocal { log.Warningf("tried to update metadata for non-local profile %s", profile.ScopedID()) return false } - profile.Lock() - defer profile.Unlock() - - // Update special profile and return if it was one. - if ok, changed := updateSpecialProfileMetadata(profile, binaryPath); ok { - return changed + // Set PresentationPath if unset. + if profile.PresentationPath == "" && binaryPath != "" { + profile.PresentationPath = binaryPath + changed = true } - var needsUpdateFromSystem bool + // Migrate LinkedPath to PresentationPath. + // TODO: Remove in v1.5 + if profile.PresentationPath == "" && profile.LinkedPath != "" { + profile.PresentationPath = profile.LinkedPath + changed = true + } - // Check profile name. - filename := filepath.Base(profile.PresentationPath) - - // Update profile name if it is empty or equals the filename, which is the - // case for older profiles. - if strings.TrimSpace(profile.Name) == "" || profile.Name == filename { - // Generate a default profile name if does not exist. + // Set Name if unset. + if profile.Name == "" && profile.PresentationPath != "" { + // Generate a default profile name from path. profile.Name = osdetail.GenerateBinaryNameFromPath(profile.PresentationPath) - if profile.Name == filename { - // TODO: Theoretically, the generated name could be identical to the - // filename. - // As a quick fix, append a space to the name. - profile.Name += " " + changed = true + } + + // Migrato to Fingerprints. + // TODO: Remove in v1.5 + if len(profile.Fingerprints) == 0 && profile.LinkedPath != "" { + profile.Fingerprints = []Fingerprint{ + { + Type: FingerprintTypePathID, + Operation: FingerprintOperationEqualsID, + Value: profile.LinkedPath, + }, } changed = true - needsUpdateFromSystem = true } - // If needed, get more/better data from the operating system. - if needsUpdateFromSystem { - module.StartWorker("get profile metadata", profile.updateMetadataFromSystem) + // UI Backward Compatibility: + // Fill LinkedPath with PresentationPath + // TODO: Remove in v1.1 + if profile.LinkedPath == "" && profile.PresentationPath != "" { + profile.LinkedPath = profile.PresentationPath + changed = true } return changed @@ -469,23 +478,14 @@ func (profile *Profile) UpdateMetadata(binaryPath string) (changed bool) { // updateMetadataFromSystem updates the profile metadata with data from the // operating system and saves it afterwards. func (profile *Profile) updateMetadataFromSystem(ctx context.Context) error { + var changed bool + // This function is only valid for local profiles. if profile.Source != SourceLocal || profile.PresentationPath == "" { return fmt.Errorf("tried to update metadata for non-local or non-path profile %s", profile.ScopedID()) } - // Save the profile when finished, if needed. - save := false - defer func() { - if save { - err := profile.Save() - if err != nil { - log.Warningf("profile: failed to save %s after metadata update: %s", profile.ScopedID(), err) - } - } - }() - - // Get binary name from linked path. + // Get binary name from PresentationPath. newName, err := osdetail.GetBinaryNameFromSystem(profile.PresentationPath) if err != nil { switch { @@ -503,25 +503,26 @@ func (profile *Profile) updateMetadataFromSystem(ctx context.Context) error { return nil } - // Get filename of linked path for comparison. - filename := filepath.Base(profile.PresentationPath) + // Apply new data to profile. + func() { + // Lock profile for applying metadata. + profile.Lock() + defer profile.Unlock() - // TODO: Theoretically, the generated name from the system could be identical - // to the filename. This would mean that the worker is triggered every time - // the profile is freshly loaded. - if newName == filename { - // As a quick fix, append a space to the name. - newName += " " - } + // Apply new name if it changed. + if profile.Name != newName { + profile.Name = newName + changed = true + } + }() - // Lock profile for applying metadata. - profile.Lock() - defer profile.Unlock() - - // Apply new name if it changed. - if profile.Name != newName { - profile.Name = newName - save = true + // If anything changed, save the profile. + // profile.Lock must not be held! + if changed { + err := profile.Save() + if err != nil { + log.Warningf("profile: failed to save %s after metadata update: %s", profile.ScopedID(), err) + } } return nil diff --git a/profile/special.go b/profile/special.go index 89b06f3c..79c2a7b5 100644 --- a/profile/special.go +++ b/profile/special.go @@ -110,20 +110,40 @@ func GetSpecialProfile(id string, path string) ( //nolint:gocognit } // Get special profile from DB and check if it needs a reset. + var created bool profile, err = getProfile(scopedID) - if err != nil { - if !errors.Is(err, database.ErrNotFound) { - log.Warningf("profile: failed to get special profile %s: %s", id, err) + switch { + case err == nil: + // Reset profile if needed. + if specialProfileNeedsReset(profile) { + profile = createSpecialProfile(id, path) + created = true } + case !errors.Is(err, database.ErrNotFound): + // Warn when fetching from DB fails, and create new profile as fallback. + log.Warningf("profile: failed to get special profile %s: %s", id, err) + fallthrough + default: + // Create new profile if it does not exist (or failed to load). profile = createSpecialProfile(id, path) - } else if specialProfileNeedsReset(profile) { - log.Debugf("profile: resetting special profile %s", id) - profile = createSpecialProfile(id, path) + created = true } + // Check if creating the special profile was successful. if profile == nil { return nil, errors.New("given ID is not a special profile ID") } + // Update metadata + changed := updateSpecialProfileMetadata(profile, path) + + // Save if created or changed. + if created || changed { + err := profile.Save() + if err != nil { + log.Warningf("profile: failed to save special profile %s: %s", scopedID, err) + } + } + // Prepare profile for first use. // If we are refetching, assign the layered profile from the previous version. @@ -144,7 +164,7 @@ func GetSpecialProfile(id string, path string) ( //nolint:gocognit return profile, nil } -func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (ok, changed bool) { +func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (changed bool) { // Get new profile name and check if profile is applicable to special handling. var newProfileName, newDescription string switch profile.ID { @@ -170,7 +190,7 @@ func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (ok, chan newProfileName = PortmasterNotifierProfileName newDescription = PortmasterNotifierProfileDescription default: - return false, false + return false } // Update profile name if needed. @@ -191,7 +211,7 @@ func updateSpecialProfileMetadata(profile *Profile, binaryPath string) (ok, chan changed = true } - return true, changed + return changed } func createSpecialProfile(profileID string, path string) *Profile { @@ -203,6 +223,13 @@ func createSpecialProfile(profileID string, path string) *Profile { PresentationPath: path, }) + case UnsolicitedProfileID: + return New(&Profile{ + ID: UnsolicitedProfileID, + Source: SourceLocal, + PresentationPath: path, + }) + case SystemProfileID: return New(&Profile{ ID: SystemProfileID, @@ -306,7 +333,7 @@ func specialProfileNeedsReset(profile *Profile) bool { switch profile.ID { case SystemResolverProfileID: - return canBeUpgraded(profile, "20.11.2021") + return canBeUpgraded(profile, "21.10.2022") case PortmasterAppProfileID: return canBeUpgraded(profile, "8.9.2021") default: From 617644c1939d6b01c717c2ac73f52f12c9d88249 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 6 Oct 2022 10:33:34 +0200 Subject: [PATCH 10/15] Add migration for profiles --- profile/migrations.go | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/profile/migrations.go b/profile/migrations.go index a28baf7e..aee673bd 100644 --- a/profile/migrations.go +++ b/profile/migrations.go @@ -2,12 +2,14 @@ package profile import ( "context" + "fmt" "github.com/hashicorp/go-version" "github.com/safing/portbase/config" "github.com/safing/portbase/database" "github.com/safing/portbase/database/migration" + "github.com/safing/portbase/database/query" "github.com/safing/portbase/log" "github.com/safing/portmaster/status" ) @@ -19,6 +21,11 @@ func registerMigrations() error { Version: "v0.7.19", MigrateFunc: migrateNetworkRatingSystem, }, + migration.Migration{ + Description: "Migrate from LinkedPath to Fingerprints and PresentationPath", + Version: "v0.9.9", + MigrateFunc: migrateLinkedPath, + }, ) } @@ -50,3 +57,43 @@ func migrateNetworkRatingSystem(ctx context.Context, _, to *version.Version, db return nil } + +func migrateLinkedPath(ctx context.Context, _, to *version.Version, db *database.Interface) error { + // Get iterator over all profiles. + it, err := db.Query(query.New(profilesDBPath)) + if err != nil { + return fmt.Errorf("failed to query profiles: %w", err) + } + + // Migrate all profiles. + for r := range it.Next { + // Parse profile. + profile, err := EnsureProfile(r) + if err != nil { + log.Tracer(ctx).Debugf("profiles: failed to parse profile %s for migration: %s", r.Key(), err) + continue + } + + // Skip if there is no LinkedPath to migrate from. + if profile.LinkedPath == "" { + continue + } + + // Update metadata and save if changed. + if profile.updateMetadata("") { + err = db.Put(profile) + if err != nil { + log.Tracer(ctx).Debugf("profiles: failed to save profile %s after migration: %s", r.Key(), err) + } else { + log.Tracer(ctx).Tracef("profiles: migrated profile %s to %s", r.Key(), to) + } + } + } + + // Check if there was an error while iterating. + if it.Err() != nil { + return fmt.Errorf("profiles: failed to iterate over profiles for migration: %w", err) + } + + return nil +} From b3007b71db31583c224d29c96f4ebc88eea07791 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 6 Oct 2022 10:53:53 +0200 Subject: [PATCH 11/15] Add default icon for Windows services --- process/tags/svchost_windows.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process/tags/svchost_windows.go b/process/tags/svchost_windows.go index 60ab951b..713c2c31 100644 --- a/process/tags/svchost_windows.go +++ b/process/tags/svchost_windows.go @@ -82,6 +82,8 @@ func (h *SVCHostTagHandler) CreateProfile(p *process.Process) *profile.Profile { return profile.New(&profile.Profile{ Source: profile.SourceLocal, Name: "Windows Service: " + osdetail.GenerateBinaryNameFromPath(tag.Value), + Icon: `C:\Windows\System32\@WLOGO_48x48.png`, + IconType: profile.IconTypeFile, UsePresentationPath: false, Fingerprints: []profile.Fingerprint{ profile.Fingerprint{ From 77c0d954a93aeb1994dce271e8063d5895ed44de Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Mon, 10 Oct 2022 15:28:57 +0200 Subject: [PATCH 12/15] Add support for cmdline matching and add basic interpreter support --- go.mod | 1 + go.sum | 2 + netquery/database.go | 14 ++ netquery/module_api.go | 2 +- package-lock.json | 6 + process/process.go | 24 +++- process/tags/appimage_unix.go | 29 ++-- process/tags/interpreter_unix.go | 220 +++++++++++++++++++++++++++++++ process/tags/svchost_windows.go | 33 +++-- process/tags/winstore_windows.go | 31 +++-- profile/fingerprint.go | 42 ++++-- 11 files changed, 338 insertions(+), 66 deletions(-) create mode 100644 package-lock.json create mode 100644 process/tags/interpreter_unix.go diff --git a/go.mod b/go.mod index 655f7855..ec371226 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/godbus/dbus/v5 v5.1.0 github.com/google/gopacket v1.1.19 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.6.0 github.com/jackc/puddle/v2 v2.0.0-beta.1 diff --git a/go.sum b/go.sum index 7d7c69c3..ddf6a3a4 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= diff --git a/netquery/database.go b/netquery/database.go index c7d8befd..194ceac6 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -193,6 +193,20 @@ func (db *Database) ApplyMigrations() error { return fmt.Errorf("failed to create schema: %w", err) } + // create a few indexes + indexes := []string{ + `CREATE INDEX profile_id_index ON %s (profile)`, + `CREATE INDEX started_time_index ON %s (strftime('%%s', started)+0)`, + `CREATE INDEX started_ended_time_index ON %s (strftime('%%s', started)+0, strftime('%%s', ended)+0) WHERE ended IS NOT NULL`, + } + for _, idx := range indexes { + stmt := fmt.Sprintf(idx, db.Schema.Name) + + if err := sqlitex.ExecuteTransient(db.writeConn, stmt, nil); err != nil { + return fmt.Errorf("failed to create index: %q: %w", idx, err) + } + } + return nil } diff --git a/netquery/module_api.go b/netquery/module_api.go index cb7d08e9..2787b260 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -101,6 +101,7 @@ func (m *module) start() error { if err != nil { return fmt.Errorf("failed to subscribe to network tree: %w", err) } + defer close(m.feed) defer func() { _ = sub.Cancel() }() @@ -162,7 +163,6 @@ func (m *module) start() error { } func (m *module) stop() error { - close(m.feed) return nil } diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..4bec6c1f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "workspace", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} diff --git a/process/process.go b/process/process.go index 3bc261bf..47764f29 100644 --- a/process/process.go +++ b/process/process.go @@ -68,6 +68,15 @@ type Process struct { ExecHashes map[string]string } +func (p *Process) GetTag(tagID string) (profile.Tag, bool) { + for _, t := range p.Tags { + if t.Key == tagID { + return t, true + } + } + return profile.Tag{}, false +} + // Profile returns the assigned layered profile. func (p *Process) Profile() *profile.LayeredProfile { if p == nil { @@ -226,11 +235,13 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { _, process.ExecName = filepath.Split(process.Path) // Current working directory - // net yet implemented for windows - // new.Cwd, err = pInfo.Cwd() - // if err != nil { - // log.Warningf("process: failed to get Cwd: %w", err) - // } + // not yet implemented for windows + if runtime.GOOS != "windows" { + process.Cwd, err = pInfo.Cwd() + if err != nil { + log.Warningf("process: failed to get Cwd: %w", err) + } + } // Command line arguments process.CmdLine, err = pInfo.CmdlineWithContext(ctx) @@ -292,3 +303,6 @@ func (md *MatchingData) Path() string { return md.p.Path } // MatchingPath returns process.MatchingPath. func (md *MatchingData) MatchingPath() string { return md.p.MatchingPath } + +// Cmdline returns the command line of the process. +func (md *MatchingData) Cmdline() string { return md.p.CmdLine } diff --git a/process/tags/appimage_unix.go b/process/tags/appimage_unix.go index 094a5f39..37548242 100644 --- a/process/tags/appimage_unix.go +++ b/process/tags/appimage_unix.go @@ -69,22 +69,21 @@ func (h *AppImageHandler) AddTags(p *process.Process) { // CreateProfile creates a profile based on the tags of the process. // Returns nil to skip. func (h *AppImageHandler) CreateProfile(p *process.Process) *profile.Profile { - for _, tag := range p.Tags { - if tag.Key == appImagePathTagKey { - return profile.New(&profile.Profile{ - Source: profile.SourceLocal, - Name: osdetail.GenerateBinaryNameFromPath(tag.Value), - PresentationPath: p.Path, - UsePresentationPath: true, - Fingerprints: []profile.Fingerprint{ - { - Type: profile.FingerprintTypePathID, - Operation: profile.FingerprintOperationEqualsID, - Value: tag.Value, // Value of appImagePathTagKey. - }, + if tag, ok := p.GetTag(appImagePathTagKey); ok { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: osdetail.GenerateBinaryNameFromPath(tag.Value), + PresentationPath: p.Path, + UsePresentationPath: true, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypePathID, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, // Value of appImagePathTagKey. }, - }) - } + }, + }) } + return nil } diff --git a/process/tags/interpreter_unix.go b/process/tags/interpreter_unix.go new file mode 100644 index 00000000..6a01e405 --- /dev/null +++ b/process/tags/interpreter_unix.go @@ -0,0 +1,220 @@ +package tags + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "unicode/utf8" + + "github.com/google/shlex" + "github.com/safing/portmaster/process" + "github.com/safing/portmaster/profile" +) + +func init() { + if err := process.RegisterTagHandler(new(InterpHandler)); err != nil { + panic(err) + } +} + +type interpType struct { + process.TagDescription + + Regex *regexp.Regexp +} + +var knownInterperters = []interpType{ + { + TagDescription: process.TagDescription{ + ID: "python-script", + Name: "Python Script", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/python[23]\.[0-9]+$`), + }, + { + TagDescription: process.TagDescription{ + ID: "shell-script", + Name: "Shell Script", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/(ba|k|z|a)?sh$`), + }, + { + TagDescription: process.TagDescription{ + ID: "perl-script", + Name: "Perl Script", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/perl$`), + }, + { + TagDescription: process.TagDescription{ + ID: "ruby-script", + Name: "Ruby Script", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/ruby$`), + }, + { + TagDescription: process.TagDescription{ + ID: "nodejs-script", + Name: "NodeJS Script", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/node(js)?$`), + }, + /* + While similar to nodejs, electron is a bit harder as it uses a multiple processes + like Chromium and thus a interpreter match on them will but those processes into + different groups. + + I'm still not sure how this could work in the future. Maybe processes should try to + inherit the profile of the parents if there is no profile that matches the current one.... + + { + TagDescription: process.TagDescription{ + ID: "electron-app", + Name: "Electron App", + }, + Regex: regexp.MustCompile(`^(/usr)?/bin/electron([0-9]+)?$`), + }, + */ +} + +func fileMustBeUTF8(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + + defer f.Close() + + // read the first chunk of bytes + buf := new(bytes.Buffer) + size, _ := io.CopyN(buf, f, 128) + if size == 0 { + return false + } + + b := buf.Bytes()[:size] + for len(b) > 0 { + r, runeSize := utf8.DecodeRune(b) + if r == utf8.RuneError { + return false + } + + b = b[runeSize:] + } + + return true +} + +type InterpHandler struct{} + +func (h *InterpHandler) Name() string { + return "Interpreter" +} + +func (h *InterpHandler) TagDescriptions() []process.TagDescription { + l := make([]process.TagDescription, len(knownInterperters)) + for idx, it := range knownInterperters { + l[idx] = it.TagDescription + } + + return l +} + +func (h *InterpHandler) CreateProfile(p *process.Process) *profile.Profile { + for _, it := range knownInterperters { + if tag, ok := p.GetTag(it.ID); ok { + // we can safely ignore the error + args, err := shlex.Split(p.CmdLine) + if err != nil { + // this should not happen since we already called shlex.Split() + // when adding the tag. Though, make the linter happy and bail out + return nil + } + + // if arg0 is the interpreter name itself strip it away + // and use the next one + if it.Regex.MatchString(args[0]) && len(args) > 1 { + args = args[1:] + } + + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: fmt.Sprintf("%s: %s", it.Name, args[0]), + PresentationPath: tag.Value, + UsePresentationPath: true, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypeTagID, + Operation: profile.FingerprintOperationEqualsID, + Key: it.ID, + Value: tag.Value, + }, + }, + }) + } + } + return nil +} + +func (h *InterpHandler) AddTags(p *process.Process) { + // check if we have a matching interpreter + var matched interpType + for _, it := range knownInterperters { + if it.Regex.MatchString(p.Path) { + matched = it + } + } + + // zero value means we did not find any interpreter matches. + if matched.ID == "" { + return + } + + args, err := shlex.Split(p.CmdLine) + if err != nil { + // give up if we failed to parse the command line + return + } + + // if args[0] matches the interpreter name we expect + // the second arg to be a file-name + if matched.Regex.MatchString(args[0]) { + if len(args) == 1 { + // there's no argument given, this is likely an interactive + // interpreter session + return + } + + filePath := args[1] + if !filepath.IsAbs(filePath) { + filePath = filepath.Join( + p.Cwd, + filePath, + ) + } + + // TODO(ppacher): there could be some other arguments as well + // so it may be better to scan the whole command line for a path to a UTF8 + // file and use that one. + if !fileMustBeUTF8(filePath) { + return + } + + p.Tags = append(p.Tags, profile.Tag{ + Key: matched.ID, + Value: filePath, + }) + + return + } + + // we know that this process is interpreted by some known interpreter but args[0] + // does not contain the path to the interpreter. + p.Tags = append(p.Tags, profile.Tag{ + Key: matched.ID, + Value: args[0], + }) +} diff --git a/process/tags/svchost_windows.go b/process/tags/svchost_windows.go index 713c2c31..f509a7e4 100644 --- a/process/tags/svchost_windows.go +++ b/process/tags/svchost_windows.go @@ -77,24 +77,23 @@ func (h *SVCHostTagHandler) AddTags(p *process.Process) { // CreateProfile creates a profile based on the tags of the process. // Returns nil to skip. func (h *SVCHostTagHandler) CreateProfile(p *process.Process) *profile.Profile { - for _, tag := range p.Tags { - if tag.Key == svchostTagKey { - return profile.New(&profile.Profile{ - Source: profile.SourceLocal, - Name: "Windows Service: " + osdetail.GenerateBinaryNameFromPath(tag.Value), - Icon: `C:\Windows\System32\@WLOGO_48x48.png`, - IconType: profile.IconTypeFile, - UsePresentationPath: false, - Fingerprints: []profile.Fingerprint{ - profile.Fingerprint{ - Type: profile.FingerprintTypeTagID, - Key: tag.Key, - Operation: profile.FingerprintOperationEqualsID, - Value: tag.Value, - }, + if tag, ok := p.GetTag(svchostTagKey); ok { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: "Windows Service: " + osdetail.GenerateBinaryNameFromPath(tag.Value), + Icon: `C:\Windows\System32\@WLOGO_48x48.png`, + IconType: profile.IconTypeFile, + UsePresentationPath: false, + Fingerprints: []profile.Fingerprint{ + profile.Fingerprint{ + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, }, - }) - } + }, + }) } + return nil } diff --git a/process/tags/winstore_windows.go b/process/tags/winstore_windows.go index bc3b538d..79200c53 100644 --- a/process/tags/winstore_windows.go +++ b/process/tags/winstore_windows.go @@ -101,23 +101,22 @@ func (h *WinStoreHandler) AddTags(p *process.Process) { // CreateProfile creates a profile based on the tags of the process. // Returns nil to skip. func (h *WinStoreHandler) CreateProfile(p *process.Process) *profile.Profile { - for _, tag := range p.Tags { - if tag.Key == winStoreAppNameTagKey { - return profile.New(&profile.Profile{ - Source: profile.SourceLocal, - Name: osdetail.GenerateBinaryNameFromPath(tag.Value), - PresentationPath: p.Path, - UsePresentationPath: true, - Fingerprints: []profile.Fingerprint{ - { - Type: profile.FingerprintTypeTagID, - Key: tag.Key, - Operation: profile.FingerprintOperationEqualsID, - Value: tag.Value, // Value of appImagePathTagKey. - }, + if tag, ok := p.GetTag(winStoreAppNameTagKey); ok { + return profile.New(&profile.Profile{ + Source: profile.SourceLocal, + Name: osdetail.GenerateBinaryNameFromPath(tag.Value), + PresentationPath: p.Path, + UsePresentationPath: true, + Fingerprints: []profile.Fingerprint{ + { + Type: profile.FingerprintTypeTagID, + Key: tag.Key, + Operation: profile.FingerprintOperationEqualsID, + Value: tag.Value, // Value of appImagePathTagKey. }, - }) - } + }, + }) } + return nil } diff --git a/profile/fingerprint.go b/profile/fingerprint.go index b0177308..64ab1203 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -11,10 +11,11 @@ import ( // There are three levels: // // 1. Type: What matched? -// 1. Tag: 40.000 points -// 2. Env: 30.000 points -// 3. MatchingPath: 20.000 points -// 4. Path: 10.000 points +// 1. Tag: 50.000 points +// 2. Cmdline: 40.000 points +// 3. Env: 30.000 points +// 4. MatchingPath: 20.000 points +// 5. Path: 10.000 points // 2. Operation: How was it mached? // 1. Equals: 3.000 points // 2. Prefix: 2.000 points @@ -32,15 +33,17 @@ import ( // Fingerprint Type IDs. const ( - FingerprintTypeTagID = "tag" - FingerprintTypeEnvID = "env" - FingerprintTypePathID = "path" // Matches both MatchingPath and Path. + FingerprintTypeTagID = "tag" + FingerprintTypeCmdlineID = "cmdline" + FingerprintTypeEnvID = "env" + FingerprintTypePathID = "path" // Matches both MatchingPath and Path. FingerprintOperationEqualsID = "equals" FingerprintOperationPrefixID = "prefix" FingerprintOperationRegexID = "regex" - tagMatchBaseScore = 40_000 + tagMatchBaseScore = 50_000 + cmdlineMatchBaseScore = 40_000 envMatchBaseScore = 30_000 matchingPathMatchBaseScore = 20_000 pathMatchBaseScore = 10_000 @@ -75,6 +78,7 @@ type ( Env() map[string]string Path() string MatchingPath() string + Cmdline() string } matchingFingerprint interface { @@ -155,9 +159,10 @@ func (fp fingerprintRegex) Match(value string) (score int) { } type parsedFingerprints struct { - tagPrints []matchingFingerprint - envPrints []matchingFingerprint - pathPrints []matchingFingerprint + tagPrints []matchingFingerprint + envPrints []matchingFingerprint + pathPrints []matchingFingerprint + cmdlinePrints []matchingFingerprint } func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed *parsedFingerprints, firstErr error) { @@ -187,7 +192,7 @@ func parseFingerprints(raw []Fingerprint, deprecatedLinkedPath string) (parsed * } continue } - case FingerprintTypePathID: + case FingerprintTypePathID, FingerprintTypeCmdlineID: // Don't need a key. default: // Unknown type. @@ -236,6 +241,8 @@ func (parsed *parsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchin parsed.envPrints = append(parsed.envPrints, matchingPrint) case FingerprintTypePathID: parsed.pathPrints = append(parsed.pathPrints, matchingPrint) + case FingerprintTypeCmdlineID: + parsed.cmdlinePrints = append(parsed.cmdlinePrints, matchingPrint) default: // This should never happen, as the types are checked already. panic(fmt.Sprintf("unknown fingerprint type: %q", fp.Type)) @@ -265,6 +272,17 @@ func MatchFingerprints(prints *parsedFingerprints, md MatchingData) (highestScor return tagMatchBaseScore + highestScore } + cmdline := md.Cmdline() + for _, cmdlinePrint := range prints.cmdlinePrints { + if score := cmdlinePrint.Match(cmdline); score > highestScore { + highestScore = score + } + + } + if highestScore > 0 { + return cmdlineMatchBaseScore + highestScore + } + // Check env. for _, envPrint := range prints.envPrints { for key, value := range md.Env() { From 0810eee7bb2118707a222d47d3e203c922b87baf Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Mon, 10 Oct 2022 18:08:09 +0200 Subject: [PATCH 13/15] Fix linter errors --- package-lock.json | 6 ------ process/process.go | 3 ++- process/tags/interpreter_unix.go | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) delete mode 100644 package-lock.json diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 4bec6c1f..00000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "workspace", - "lockfileVersion": 2, - "requires": true, - "packages": {} -} diff --git a/process/process.go b/process/process.go index 47764f29..a3ff1333 100644 --- a/process/process.go +++ b/process/process.go @@ -68,6 +68,7 @@ type Process struct { ExecHashes map[string]string } +// GetTag returns the process tag with the given ID. func (p *Process) GetTag(tagID string) (profile.Tag, bool) { for _, t := range p.Tags { if t.Key == tagID { @@ -239,7 +240,7 @@ func loadProcess(ctx context.Context, pid int) (*Process, error) { if runtime.GOOS != "windows" { process.Cwd, err = pInfo.Cwd() if err != nil { - log.Warningf("process: failed to get Cwd: %w", err) + log.Warningf("process: failed to get Cwd: %s", err) } } diff --git a/process/tags/interpreter_unix.go b/process/tags/interpreter_unix.go index 6a01e405..0afc0357 100644 --- a/process/tags/interpreter_unix.go +++ b/process/tags/interpreter_unix.go @@ -86,7 +86,9 @@ func fileMustBeUTF8(path string) bool { return false } - defer f.Close() + defer func() { + _ = f.Close() + }() // read the first chunk of bytes buf := new(bytes.Buffer) @@ -108,12 +110,15 @@ func fileMustBeUTF8(path string) bool { return true } +// InterpHandler supports adding process tags based on well-known interpreter binaries. type InterpHandler struct{} +// Name returns "Interpreter" func (h *InterpHandler) Name() string { return "Interpreter" } +// TagDescriptions returns a set of tag descriptions that InterpHandler provides. func (h *InterpHandler) TagDescriptions() []process.TagDescription { l := make([]process.TagDescription, len(knownInterperters)) for idx, it := range knownInterperters { @@ -123,6 +128,8 @@ func (h *InterpHandler) TagDescriptions() []process.TagDescription { return l } +// CreateProfile creates a new profile for any process that has a tag created +// by InterpHandler. func (h *InterpHandler) CreateProfile(p *process.Process) *profile.Profile { for _, it := range knownInterperters { if tag, ok := p.GetTag(it.ID); ok { @@ -159,6 +166,8 @@ func (h *InterpHandler) CreateProfile(p *process.Process) *profile.Profile { return nil } +// AddTags inspects the process p and adds any interpreter tags that InterpHandler +// detects. func (h *InterpHandler) AddTags(p *process.Process) { // check if we have a matching interpreter var matched interpType @@ -208,6 +217,8 @@ func (h *InterpHandler) AddTags(p *process.Process) { Value: filePath, }) + p.MatchingPath = filePath + return } @@ -217,4 +228,6 @@ func (h *InterpHandler) AddTags(p *process.Process) { Key: matched.ID, Value: args[0], }) + + p.MatchingPath = args[0] } From 3c8157fd91fe90a49d3e82686a1b6af1e3236d42 Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Mon, 10 Oct 2022 18:09:40 +0200 Subject: [PATCH 14/15] Implement review changes --- process/tags/interpreter_unix.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/process/tags/interpreter_unix.go b/process/tags/interpreter_unix.go index 0afc0357..237d8eeb 100644 --- a/process/tags/interpreter_unix.go +++ b/process/tags/interpreter_unix.go @@ -197,27 +197,27 @@ func (h *InterpHandler) AddTags(p *process.Process) { return } - filePath := args[1] - if !filepath.IsAbs(filePath) { - filePath = filepath.Join( + scriptPath := args[1] + if !filepath.IsAbs(scriptPath) { + scriptPath = filepath.Join( p.Cwd, - filePath, + scriptPath, ) } // TODO(ppacher): there could be some other arguments as well // so it may be better to scan the whole command line for a path to a UTF8 // file and use that one. - if !fileMustBeUTF8(filePath) { + if !fileMustBeUTF8(scriptPath) { return } p.Tags = append(p.Tags, profile.Tag{ Key: matched.ID, - Value: filePath, + Value: scriptPath, }) - p.MatchingPath = filePath + p.MatchingPath = scriptPath return } From 144e5d831262ba670c8b268585361e734ec26a19 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 10 Oct 2022 22:57:27 +0200 Subject: [PATCH 15/15] Fix linter errors --- netquery/module_api.go | 1 - process/tags/interpreter_unix.go | 3 +- profile/fingerprint.go | 57 ++++++++++++++++---------------- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/netquery/module_api.go b/netquery/module_api.go index 2787b260..4cb02462 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -163,6 +163,5 @@ func (m *module) start() error { } func (m *module) stop() error { - return nil } diff --git a/process/tags/interpreter_unix.go b/process/tags/interpreter_unix.go index 237d8eeb..c254f4a7 100644 --- a/process/tags/interpreter_unix.go +++ b/process/tags/interpreter_unix.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "github.com/google/shlex" + "github.com/safing/portmaster/process" "github.com/safing/portmaster/profile" ) @@ -113,7 +114,7 @@ func fileMustBeUTF8(path string) bool { // InterpHandler supports adding process tags based on well-known interpreter binaries. type InterpHandler struct{} -// Name returns "Interpreter" +// Name returns "Interpreter". func (h *InterpHandler) Name() string { return "Interpreter" } diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 64ab1203..6f5e6389 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -25,12 +25,6 @@ import ( // 2. Prefix: Length of prefix // 3. Regex: Length of match -// ms-store:Microsoft.One.Note - -// Path Match /path/to/file -// Tag MS-Store Match value -// Env Regex Key Value - // Fingerprint Type IDs. const ( FingerprintTypeTagID = "tag" @@ -253,34 +247,39 @@ func (parsed *parsedFingerprints) addMatchingFingerprint(fp Fingerprint, matchin // fingerprints and matching data. func MatchFingerprints(prints *parsedFingerprints, md MatchingData) (highestScore int) { // Check tags. - for _, tagPrint := range prints.tagPrints { - for _, tag := range md.Tags() { - // Check if tag key matches. - if !tagPrint.MatchesKey(tag.Key) { - continue - } + tags := md.Tags() + if len(tags) > 0 { + for _, tagPrint := range prints.tagPrints { + for _, tag := range tags { + // Check if tag key matches. + if !tagPrint.MatchesKey(tag.Key) { + continue + } - // Try matching the tag value. - score := tagPrint.Match(tag.Value) - if score > highestScore { + // Try matching the tag value. + score := tagPrint.Match(tag.Value) + if score > highestScore { + highestScore = score + } + } + } + // If something matched, add base score and return. + if highestScore > 0 { + return tagMatchBaseScore + highestScore + } + } + + // Check cmdline. + cmdline := md.Cmdline() + if cmdline != "" { + for _, cmdlinePrint := range prints.cmdlinePrints { + if score := cmdlinePrint.Match(cmdline); score > highestScore { highestScore = score } } - } - // If something matched, add base score and return. - if highestScore > 0 { - return tagMatchBaseScore + highestScore - } - - cmdline := md.Cmdline() - for _, cmdlinePrint := range prints.cmdlinePrints { - if score := cmdlinePrint.Match(cmdline); score > highestScore { - highestScore = score + if highestScore > 0 { + return cmdlineMatchBaseScore + highestScore } - - } - if highestScore > 0 { - return cmdlineMatchBaseScore + highestScore } // Check env.