github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1313 from safing/feature/profile-sync

Browse Source 
Add profile merging and prepare for sync
This commit is contained in:
Daniel Hovie
2023-09-19 17:08:37 +02:00
committed by GitHub
parent bd410afee6 9bb897aa89
commit 6ff7c5f7b6
14 changed files with 502 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/base/logs.go
View File

@@ -2,6 +2,7 @@ package base
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
@@ -31,7 +32,9 @@ func logCleaner(_ context.Context, _ *modules.Task) error {
filepath.Join(dataroot.Root().Path, logFileDir),
func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Warningf("core: failed to access %s while deleting old log files: %s", path, err)
if !errors.Is(err, os.ErrNotExist) {
log.Warningf("core: failed to access %s while deleting old log files: %s", path, err)
}
return nil
}

20
firewall/master.go
View File

@@ -33,6 +33,7 @@ var defaultDeciders = []deciderFn{
checkConnectionType,
checkConnectionScope,
checkEndpointLists,
checkInvalidIP,
checkResolverScope,
checkConnectivityDomain,
checkBypassPrevention,
@@ -371,7 +372,8 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil
return true
}
case netutils.Undefined, netutils.Invalid:
fallthrough
// Block Invalid / Undefined IPs _after_ the rules.
return false
default:
conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound
return true
@@ -380,6 +382,22 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil
return false
}
func checkInvalidIP(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
// Only applies to IP connections.
if conn.Type != network.IPConnection {
return false
}
// Block Invalid / Undefined IPs.
switch conn.Entity.IPScope { //nolint:exhaustive // Only looking for specific values.
case netutils.Undefined, netutils.Invalid:
conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound
return true
}
return false
}
func checkBypassPrevention(ctx context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
if p.PreventBypassing() {
// check for bypass protection

2
firewall/tunnel.go
View File

@@ -160,6 +160,8 @@ func DeriveTunnelOptions(lp *profile.LayeredProfile, proc *process.Process, dest
}
if !connEncrypted {
tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateTrusted)
// TODO: Add this when all Hubs are on v0.6.21+
// tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateAllowUnencrypted)
}
// Add required verified owners if community nodes should not be used.

4
go.mod
View File

@@ -20,9 +20,9 @@ require (
github.com/mitchellh/go-server-timing v1.0.1
github.com/oschwald/maxminddb-golang v1.12.0
github.com/safing/jess v0.3.1
github.com/safing/portbase v0.17.4
github.com/safing/portbase v0.17.5
github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec
github.com/safing/spn v0.6.19
github.com/safing/spn v0.6.21
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/spf13/cobra v1.7.0
github.com/spkg/zipfs v0.7.1

8
go.sum
View File

@@ -240,12 +240,12 @@ github.com/safing/jess v0.3.1 h1:cMZVhi2whW/YdD98MPLeLIWJndQ7o2QVt2HefQ/ByFA=
github.com/safing/jess v0.3.1/go.mod h1:aj73Eot1zm2ETkJuw9hJlIO8bRom52uBbsCHemvlZmA=
github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y=
github.com/safing/portbase v0.16.2/go.mod h1:mzNCWqPbO7vIYbbK5PElGbudwd2vx4YPNawymL8Aro8=
github.com/safing/portbase v0.17.4 h1:4RhItvFujwdfLQVfwvB+VYER33AT//Ywv317Vj01TEQ=
github.com/safing/portbase v0.17.4/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI=
github.com/safing/portbase v0.17.5 h1:0gq0tgPLbKlK+xq7WM+Kcutu5HgYIglxBE3QqN5tIAA=
github.com/safing/portbase v0.17.5/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI=
github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec h1:oSJY1seobofPwpMoJRkCgXnTwfiQWNfGMCPDfqgAEfg=
github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE=
github.com/safing/spn v0.6.19 h1:z4i8hb5FGKjmgSzA4MzJ8mOc0hYp11zgXzujrHwwV5k=
github.com/safing/spn v0.6.19/go.mod h1:LRWLManSXHTViiDqU2qNy3w07auMuadOnVW8wAB/Cgw=
github.com/safing/spn v0.6.21 h1:7LhaEbQ7xrPMETerydpbEAVmLmp+etGJWKnW5b6iI0g=
github.com/safing/spn v0.6.21/go.mod h1:MgWfUDkYqi46A+EcxayLD0tc519KBiVEQ6mfAjHIx/4=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0=

6
network/connection.go
View File

@@ -488,6 +488,12 @@ func (conn *Connection) GatherConnectionInfo(pkt packet.Packet) (err error) {
// Errors are informational and are logged to the context.
}
// Only get process and profile with first real packet.
// TODO: Remove when we got full VM/Docker support.
if pkt.InfoOnly() {
return nil
}
// Get Process and Profile.
if conn.process == nil {
conn.process, err = process.GetProcessWithProfile(pkt.Ctx(), conn.PID)

86
profile/fingerprint.go
View File

@@ -4,6 +4,11 @@ import (
"fmt"
"regexp"
"strings"
"golang.org/x/exp/slices"
"github.com/safing/jess/lhash"
"github.com/safing/portbase/container"
)
// # Matching and Scores
@@ -57,6 +62,12 @@ type (
Key string // Key must always fully match.
Operation string
Value string
// MergedFrom holds the ID of the profile from which this fingerprint was
// merged from. The merged profile should create a new profile ID derived
// from the new fingerprints and add all fingerprints with this field set
// to the originating profile ID
MergedFrom string
}
// Tag represents a simple key/value kind of tag used in process metadata
@@ -347,3 +358,78 @@ func checkMatchStrength(value int) int {
}
return value
}
const (
deriveFPKeyIDForItemStart = iota + 1
deriveFPKeyIDForType
deriveFPKeyIDForKey
deriveFPKeyIDForOperation
deriveFPKeyIDForValue
)
func deriveProfileID(fps []Fingerprint) string {
// Sort the fingerprints.
sortAndCompactFingerprints(fps)
// Compile data for hashing.
c := container.New(nil)
c.AppendInt(len(fps))
for _, fp := range fps {
c.AppendNumber(deriveFPKeyIDForItemStart)
if fp.Type != "" {
c.AppendNumber(deriveFPKeyIDForType)
c.AppendAsBlock([]byte(fp.Type))
}
if fp.Key != "" {
c.AppendNumber(deriveFPKeyIDForKey)
c.AppendAsBlock([]byte(fp.Key))
}
if fp.Operation != "" {
c.AppendNumber(deriveFPKeyIDForOperation)
c.AppendAsBlock([]byte(fp.Operation))
}
if fp.Value != "" {
c.AppendNumber(deriveFPKeyIDForValue)
c.AppendAsBlock([]byte(fp.Value))
}
}
// Hash and return.
h := lhash.Digest(lhash.SHA3_256, c.CompileData())
return h.Base58()
}
func sortAndCompactFingerprints(fps []Fingerprint) []Fingerprint {
// Sort.
slices.SortFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) int {
switch {
case a.Type != b.Type:
return strings.Compare(a.Type, b.Type)
case a.Key != b.Key:
return strings.Compare(a.Key, b.Key)
case a.Operation != b.Operation:
return strings.Compare(a.Operation, b.Operation)
case a.Value != b.Value:
return strings.Compare(a.Value, b.Value)
case a.MergedFrom != b.MergedFrom:
return strings.Compare(a.MergedFrom, b.MergedFrom)
default:
return 0
}
})
// De-duplicate.
// Important: Even if the fingerprint is the same, but MergedFrom is
// different, we need to keep the separate fingerprint, so that new installs
// will cleanly update to the synced state: Auto-generated profiles need to
// be automatically replaced by the merged version.
fps = slices.CompactFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) bool {
return a.Type == b.Type &&
a.Key == b.Key &&
a.Operation == b.Operation &&
a.Value == b.Value &&
a.MergedFrom == b.MergedFrom
})
return fps
}

53
profile/fingerprint_test.go Normal file
View File

@@ -0,0 +1,53 @@
package profile
import (
"math/rand"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestDeriveProfileID(t *testing.T) {
t.Parallel()
fps := []Fingerprint{
{
Type: FingerprintTypePathID,
Operation: FingerprintOperationEqualsID,
Value: "/sbin/init",
},
{
Type: FingerprintTypePathID,
Operation: FingerprintOperationPrefixID,
Value: "/",
},
{
Type: FingerprintTypeEnvID,
Key: "PORTMASTER_PROFILE",
Operation: FingerprintOperationEqualsID,
Value: "TEST-1",
},
{
Type: FingerprintTypeTagID,
Key: "tag-key-1",
Operation: FingerprintOperationEqualsID,
Value: "tag-key-2",
},
}
// Create rand source for shuffling.
rnd := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
// Test 100 times.
for i := 0; i < 100; i++ {
// Shuffle fingerprints.
rnd.Shuffle(len(fps), func(i, j int) {
fps[i], fps[j] = fps[j], fps[i]
})
// Check if fingerprint matches.
id := deriveProfileID(fps)
assert.Equal(t, "PTSRP7rdCnmvdjRoPMTrtjj7qk7PxR1a9YdBWUGwnZXJh2", id)
}
}

57
profile/icon.go Normal file
View File

@@ -0,0 +1,57 @@
package profile
import (
"strings"
"golang.org/x/exp/slices"
)
// Icon describes an icon.
type Icon struct {
Type IconType
Value string
}
// IconType describes the type of an Icon.
type IconType string
// Supported icon types.
const (
IconTypeFile IconType = "path"
IconTypeDatabase IconType = "database"
)
func (t IconType) sortOrder() int {
switch t {
case IconTypeDatabase:
return 1
case IconTypeFile:
return 2
default:
return 100
}
}
func sortAndCompactIcons(icons []Icon) []Icon {
// Sort.
slices.SortFunc[[]Icon, Icon](icons, func(a, b Icon) int {
aOrder := a.Type.sortOrder()
bOrder := b.Type.sortOrder()
switch {
case aOrder != bOrder:
return aOrder - bOrder
case a.Value != b.Value:
return strings.Compare(a.Value, b.Value)
default:
return 0
}
})
// De-duplicate.
icons = slices.CompactFunc[[]Icon, Icon](icons, func(a, b Icon) bool {
return a.Type == b.Type && a.Value == b.Value
})
return icons
}

84
profile/merge.go Normal file
View File

@@ -0,0 +1,84 @@
package profile
import (
"fmt"
"sync"
"github.com/safing/portbase/database/record"
)
// MergeProfiles merges multiple profiles into a new one.
// The new profile is saved and returned.
// Only the icon and fingerprints are inherited from other profiles.
// All other information is taken only from the primary profile.
func MergeProfiles(primary *Profile, secondaries ...*Profile) (newProfile *Profile, err error) {
// Fill info from primary profile.
newProfile = &Profile{
Base: record.Base{},
RWMutex: sync.RWMutex{},
ID: "", // Omit ID to derive it from the new fingerprints.
Source: primary.Source,
Name: primary.Name,
Description: primary.Description,
Homepage: primary.Homepage,
UsePresentationPath: false, // Disable presentation path.
SecurityLevel: primary.SecurityLevel,
Config: primary.Config,
}
// Collect all icons.
newProfile.Icons = make([]Icon, 0, len(secondaries)+1) // Guess the needed space.
newProfile.Icons = append(newProfile.Icons, primary.Icons...)
for _, sp := range secondaries {
newProfile.Icons = append(newProfile.Icons, sp.Icons...)
}
newProfile.Icons = sortAndCompactIcons(newProfile.Icons)
// Collect all fingerprints.
newProfile.Fingerprints = make([]Fingerprint, 0, len(secondaries)+1) // Guess the needed space.
newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, primary.Fingerprints, primary.ScopedID())
for _, sp := range secondaries {
newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, sp.Fingerprints, sp.ScopedID())
}
newProfile.Fingerprints = sortAndCompactFingerprints(newProfile.Fingerprints)
// Save new profile.
newProfile = New(newProfile)
err = newProfile.Save()
if err != nil {
return nil, fmt.Errorf("failed to save merged profile: %w", err)
}
// FIXME: Should we ... ?
// newProfile.updateMetadata()
// newProfile.updateMetadataFromSystem()
// Delete all previous profiles.
// FIXME:
/*
primary.Meta().Delete()
// Set as outdated and remove from active profiles.
// Signify that profile was deleted and save for sync.
for _, sp := range secondaries {
sp.Meta().Delete()
// Set as outdated and remove from active profiles.
// Signify that profile was deleted and save for sync.
}
*/
return newProfile, nil
}
func addFingerprints(existing, add []Fingerprint, from string) []Fingerprint {
// Copy all fingerprints and add the they are from.
for _, addFP := range add {
existing = append(existing, Fingerprint{
Type: addFP.Type,
Key: addFP.Key,
Operation: addFP.Operation,
Value: addFP.Value,
MergedFrom: from,
})
}
return existing
}

163
profile/migrations.go
View File

@@ -2,6 +2,8 @@ package profile
import (
"context"
"fmt"
"regexp"
"github.com/hashicorp/go-version"
@@ -25,6 +27,16 @@ func registerMigrations() error {
Version: "v0.9.9",
MigrateFunc: migrateLinkedPath,
},
migration.Migration{
Description: "Migrate from Icon Fields to Icon List",
Version: "v1.4.7",
MigrateFunc: migrateIcons,
},
// migration.Migration{
// Description: "Migrate from random profile IDs to fingerprint-derived IDs",
// Version: "v1.5.1",
// MigrateFunc: migrateToDerivedIDs,
// },
)
}
@@ -97,3 +109,154 @@ func migrateLinkedPath(ctx context.Context, _, to *version.Version, db *database
return nil
}
func migrateIcons(ctx context.Context, _, to *version.Version, db *database.Interface) error {
// Get iterator over all profiles.
it, err := db.Query(query.New(profilesDBPath))
if err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate from icon fields: failed to start query: %s", err)
return nil
}
// Migrate all profiles.
var (
lastErr error
failed int
total int
)
for r := range it.Next {
// Parse profile.
profile, err := EnsureProfile(r)
if err != nil {
log.Tracer(ctx).Debugf("profiles: failed to parse profile %s for migration: %s", r.Key(), err)
continue
}
// Skip if there is no (valid) icon defined or the icon list is already populated.
if profile.Icon == "" || profile.IconType == "" || len(profile.Icons) > 0 {
continue
}
// Migrate to icon list.
profile.Icons = []Icon{{
Type: profile.IconType,
Value: profile.Icon,
}}
// Save back to DB.
err = db.Put(profile)
if err != nil {
failed++
lastErr = err
log.Tracer(ctx).Debugf("profiles: failed to save profile %s after migration: %s", r.Key(), err)
} else {
log.Tracer(ctx).Tracef("profiles: migrated profile %s to %s", r.Key(), to)
}
total++
}
// Check if there was an error while iterating.
if err := it.Err(); err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate from icon fields: failed to iterate over profiles for migration: %s", err)
}
// Log migration failure and try again next time.
if lastErr != nil {
// Normally, an icon migration would not be such a big error, but this is a test
// run for the profile IDs and we absolutely need to know if anything went wrong.
module.Error(
"migration-failed",
"Profile Migration Failed",
fmt.Sprintf("Failed to migrate icons of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
)
return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr)
}
return lastErr
}
var randomUUIDRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *database.Interface) error {
var profilesToDelete []string //nolint:prealloc // We don't know how many profiles there are.
// Get iterator over all profiles.
it, err := db.Query(query.New(profilesDBPath))
if err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate to derived profile IDs: failed to start query: %s", err)
return nil
}
// Migrate all profiles.
var (
lastErr error
failed int
total int
)
for r := range it.Next {
// Parse profile.
profile, err := EnsureProfile(r)
if err != nil {
log.Tracer(ctx).Debugf("profiles: failed to parse profile %s for migration: %s", r.Key(), err)
continue
}
// Skip if the ID does not look like a random UUID.
if !randomUUIDRegex.MatchString(profile.ID) {
continue
}
// Generate new ID.
oldScopedID := profile.ScopedID()
newID := deriveProfileID(profile.Fingerprints)
// If they match, skip migration for this profile.
if profile.ID == newID {
continue
}
// Reset key.
profile.ResetKey()
// Set new ID and rebuild the key.
profile.ID = newID
profile.makeKey()
// Save back to DB.
err = db.Put(profile)
if err != nil {
failed++
lastErr = err
log.Tracer(ctx).Debugf("profiles: failed to save profile %s after migration: %s", r.Key(), err)
} else {
log.Tracer(ctx).Tracef("profiles: migrated profile %s to %s", r.Key(), to)
// Add old ID to profiles that we need to delete.
profilesToDelete = append(profilesToDelete, oldScopedID)
}
total++
}
// Check if there was an error while iterating.
if err := it.Err(); err != nil {
log.Tracer(ctx).Errorf("profile: failed to migrate to derived profile IDs: failed to iterate over profiles for migration: %s", err)
}
// Delete old migrated profiles.
for _, scopedID := range profilesToDelete {
if err := db.Delete(profilesDBPath + scopedID); err != nil {
log.Tracer(ctx).Errorf("profile: failed to delete old profile %s during migration: %s", scopedID, err)
}
}
// Log migration failure and try again next time.
if lastErr != nil {
module.Error(
"migration-failed",
"Profile Migration Failed",
fmt.Sprintf("Failed to migrate profile IDs of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr),
)
return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr)
}
return nil
}

2
profile/module.go
View File

@@ -48,7 +48,7 @@ func start() error {
}
if err := migrations.Migrate(module.Ctx); err != nil {
return err
log.Errorf("profile: migrations failed: %s", err)
}
err := registerValidationDBHook()

32
profile/profile.go
View File

@@ -37,17 +37,6 @@ const (
DefaultActionPermit uint8 = 3
)
// iconType describes the type of the Icon property
// of a profile.
type iconType string
// Supported icon types.
const (
IconTypeFile iconType = "path"
IconTypeDatabase iconType = "database"
IconTypeBlob iconType = "blob"
)
// Profile is used to predefine a security profile for applications.
type Profile struct { //nolint:maligned // not worth the effort
record.Base
@@ -73,12 +62,16 @@ type Profile struct { //nolint:maligned // not worth the effort
// Homepage may refer to the website of the application
// vendor.
Homepage string
// Icon holds the icon of the application. The value
// Deprecated: Icon holds the icon of the application. The value
// may either be a filepath, a database key or a blob URL.
// See IconType for more information.
Icon string
// IconType describes the type of the Icon property.
IconType iconType
// Deprecated: IconType describes the type of the Icon property.
IconType IconType
// Icons holds a list of icons to represent the application.
Icons []Icon
// Deprecated: LinkedPath used to point to the executableis this
// profile was created for.
// Until removed, it will be added to the Fingerprints as an exact path match.
@@ -265,9 +258,16 @@ func New(profile *Profile) *Profile {
profile.Config = make(map[string]interface{})
}
// Generate random ID if none is given.
// Generate ID if none is given.
if profile.ID == "" {
profile.ID = utils.RandomUUID("").String()
if len(profile.Fingerprints) > 0 {
// Derive from fingerprints.
profile.ID = deriveProfileID(profile.Fingerprints)
} else {
// Generate random ID as fallback.
log.Warningf("profile: creating new profile without fingerprints to derive ID from")
profile.ID = utils.RandomUUID("").String()
}
}
// Make key from ID and source.

7
resolver/scopes.go
View File

@@ -171,8 +171,11 @@ func GetResolversInScope(ctx context.Context, q *Query) (selected []*Resolver, p
// Special connectivity domains
if netenv.IsConnectivityDomain(q.FQDN) && len(systemResolvers) > 0 {
// Do not do compliance checks for connectivity domains.
selected = append(selected, systemResolvers...) // dhcp assigned resolvers
selected = addResolvers(ctx, q, selected, systemResolvers)
if len(selected) == 0 {
selected = addResolvers(ctx, q, selected, localResolvers)
selected = addResolvers(ctx, q, selected, globalResolvers)
}
return selected, ServerSourceOperatingSystem, false
}