From e9e9b543644267f70732a85370e3b4dbbc212fcb Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Wed, 14 Jun 2023 09:38:25 +0200 Subject: [PATCH 01/15] Research on possible history module implementation using sqlite ATTACH DATABASE --- netquery/database.go | 103 ++++++++++++++++++++------------- netquery/orm/schema_builder.go | 9 ++- netquery/query_handler.go | 5 +- 3 files changed, 73 insertions(+), 44 deletions(-) diff --git a/netquery/database.go b/netquery/database.go index 0434d3c1..dab486c9 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -22,7 +22,7 @@ import ( ) // InMemory is the "file path" to open a new in-memory database. -const InMemory = "file:inmem.db" +const InMemory = "file:inmem.db?mode=memory" // Available connection types as their string representation. const ( @@ -115,13 +115,17 @@ func New(path string) (*Database, error) { sqlite.OpenReadOnly, sqlite.OpenNoMutex, //nolint:staticcheck // We like to be explicit. sqlite.OpenSharedCache, - sqlite.OpenMemory, + //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { return nil, fmt.Errorf("failed to open read-only sqlite connection at %s: %w", path, err) } + if err := sqlitex.ExecuteTransient(c, "ATTACH DATABASE 'file:///tmp/history.db?mode=ro' AS history", nil); err != nil { + return nil, fmt.Errorf("failed to attach history database: %w", err) + } + return c, nil } @@ -152,7 +156,7 @@ func New(path string) (*Database, error) { sqlite.OpenNoMutex, //nolint:staticcheck // We like to be explicit. sqlite.OpenWAL, sqlite.OpenSharedCache, - sqlite.OpenMemory, + //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { @@ -189,28 +193,44 @@ func NewInMemory() (*Database, error) { // any data-migrations. Once the history module is implemented this should // become/use a full migration system -- use zombiezen.com/go/sqlite/sqlitemigration. func (db *Database) ApplyMigrations() error { - // get the create-table SQL statement from the inferred schema - sql := db.Schema.CreateStatement(true) - + log.Errorf("applying migrations ...") db.l.Lock() defer db.l.Unlock() - // execute the SQL - if err := sqlitex.ExecuteTransient(db.writeConn, sql, nil); err != nil { - return fmt.Errorf("failed to create schema: %w", err) + // Attach the history database + log.Errorf("attaching database history") + if err := sqlitex.ExecuteTransient(db.writeConn, "ATTACH DATABASE 'file:///tmp/history.db?mode=rwc' AS 'history';", nil); err != nil { + return fmt.Errorf("failed to attach history database: %w", err) } - // create a few indexes - indexes := []string{ - `CREATE INDEX profile_id_index ON %s (profile)`, - `CREATE INDEX started_time_index ON %s (strftime('%%s', started)+0)`, - `CREATE INDEX started_ended_time_index ON %s (strftime('%%s', started)+0, strftime('%%s', ended)+0) WHERE ended IS NOT NULL`, - } - for _, idx := range indexes { - stmt := fmt.Sprintf(idx, db.Schema.Name) + dbNames := []string{"main", "history"} + for _, dbName := range dbNames { + // get the create-table SQL statement from the inferred schema + sql := db.Schema.CreateStatement(dbName, true) + log.Errorf("creating table schema for database %q", dbName) - if err := sqlitex.ExecuteTransient(db.writeConn, stmt, nil); err != nil { - return fmt.Errorf("failed to create index: %q: %w", idx, err) + // execute the SQL + if err := sqlitex.ExecuteTransient(db.writeConn, sql, nil); err != nil { + return fmt.Errorf("failed to create schema on database %q: %w", dbName, err) + } + + // create a few indexes + indexes := []string{ + `CREATE INDEX IF NOT EXISTS %sprofile_id_index ON %s (profile)`, + `CREATE INDEX IF NOT EXISTS %sstarted_time_index ON %s (strftime('%%s', started)+0)`, + `CREATE INDEX IF NOT EXISTS %sstarted_ended_time_index ON %s (strftime('%%s', started)+0, strftime('%%s', ended)+0) WHERE ended IS NOT NULL`, + } + for _, idx := range indexes { + name := "" + if dbName != "" { + name = dbName + "." + } + + stmt := fmt.Sprintf(idx, name, db.Schema.Name) + + if err := sqlitex.ExecuteTransient(db.writeConn, stmt, nil); err != nil { + return fmt.Errorf("failed to create index on database %q: %q: %w", dbName, idx, err) + } } } @@ -254,7 +274,7 @@ func (db *Database) CountRows(ctx context.Context) (int, error) { Count int `sqlite:"count"` } - if err := db.Execute(ctx, "SELECT COUNT(*) AS count FROM connections", orm.WithResult(&result)); err != nil { + if err := db.Execute(ctx, "SELECT COUNT(*) AS count FROM (SELECT * FROM main.connections UNION SELECT * from history.connections)", orm.WithResult(&result)); err != nil { return 0, fmt.Errorf("failed to perform query: %w", err) } @@ -273,7 +293,7 @@ func (db *Database) CountRows(ctx context.Context) (int, error) { func (db *Database) Cleanup(ctx context.Context, threshold time.Time) (int, error) { where := `WHERE ended IS NOT NULL AND datetime(ended) < datetime(:threshold)` - sql := "DELETE FROM connections " + where + ";" + sql := "DELETE FROM main.connections " + where + ";" args := orm.WithNamedArgs(map[string]interface{}{ ":threshold": threshold.UTC().Format(orm.SqliteTimeFormat), @@ -367,26 +387,29 @@ func (db *Database) Save(ctx context.Context, conn Conn) error { // TODO(ppacher): make sure this one can be cached to speed up inserting // and save some CPU cycles for the user - sql := fmt.Sprintf( - `INSERT INTO connections (%s) - VALUES(%s) - ON CONFLICT(id) DO UPDATE SET - %s - `, - strings.Join(columns, ", "), - strings.Join(placeholders, ", "), - strings.Join(updateSets, ", "), - ) + for _, dbName := range []string{"main", "history"} { + sql := fmt.Sprintf( + `INSERT INTO %s.connections (%s) + VALUES(%s) + ON CONFLICT(id) DO UPDATE SET + %s + `, + dbName, + strings.Join(columns, ", "), + strings.Join(placeholders, ", "), + strings.Join(updateSets, ", "), + ) - if err := sqlitex.Execute(db.writeConn, sql, &sqlitex.ExecOptions{ - Named: values, - ResultFunc: func(stmt *sqlite.Stmt) error { - log.Errorf("netquery: got result statement with %d columns", stmt.ColumnCount()) - return nil - }, - }); err != nil { - log.Errorf("netquery: failed to execute:\n\t%q\n\treturned error was: %s\n\tparameters: %+v", sql, err, values) - return err + if err := sqlitex.Execute(db.writeConn, sql, &sqlitex.ExecOptions{ + Named: values, + ResultFunc: func(stmt *sqlite.Stmt) error { + log.Errorf("netquery: got result statement with %d columns", stmt.ColumnCount()) + return nil + }, + }); err != nil { + log.Errorf("netquery: failed to execute:\n\t%q\n\treturned error was: %s\n\tparameters: %+v", sql, err, values) + return err + } } return nil diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go index 508b7b18..e9e46874 100644 --- a/netquery/orm/schema_builder.go +++ b/netquery/orm/schema_builder.go @@ -66,12 +66,17 @@ func (ts TableSchema) GetColumnDef(name string) *ColumnDef { } // CreateStatement build the CREATE SQL statement for the table. -func (ts TableSchema) CreateStatement(ifNotExists bool) string { +func (ts TableSchema) CreateStatement(databaseName string, ifNotExists bool) string { sql := "CREATE TABLE" if ifNotExists { sql += " IF NOT EXISTS" } - sql += " " + ts.Name + " ( " + name := ts.Name + if databaseName != "" { + name = databaseName + "." + ts.Name + } + + sql += " " + name + " ( " for idx, col := range ts.Columns { sql += col.AsSQL() diff --git a/netquery/query_handler.go b/netquery/query_handler.go index 599c71ec..e03f6a83 100644 --- a/netquery/query_handler.go +++ b/netquery/query_handler.go @@ -190,7 +190,7 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab } selectClause := req.generateSelectClause() - query := `SELECT ` + selectClause + ` FROM connections` + query := `SELECT ` + selectClause + ` FROM ( SELECT *, 'memory' as _source FROM main.connections UNION SELECT *, 'history' as _source FROM history.connections) ` if whereClause != "" { query += " WHERE " + whereClause } @@ -298,7 +298,8 @@ func (req *QueryRequestPayload) generateGroupByClause(schema *orm.TableSchema) ( func (req *QueryRequestPayload) generateSelectClause() string { selectClause := "*" if len(req.selectedFields) > 0 { - selectClause = strings.Join(req.selectedFields, ", ") + selectedFields := append(req.selectedFields, "_source") + selectClause = strings.Join(selectedFields, ", ") } return selectClause From 135b68c008ca2c94038adb39607a091f47107774 Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Wed, 14 Jun 2023 09:48:16 +0200 Subject: [PATCH 02/15] Better utilize database indexes for UNION selects --- netquery/query_handler.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/netquery/query_handler.go b/netquery/query_handler.go index e03f6a83..141f7dd5 100644 --- a/netquery/query_handler.go +++ b/netquery/query_handler.go @@ -190,11 +190,18 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab } selectClause := req.generateSelectClause() - query := `SELECT ` + selectClause + ` FROM ( SELECT *, 'memory' as _source FROM main.connections UNION SELECT *, 'history' as _source FROM history.connections) ` + inMem := `SELECT *, 'live' as _source FROM main.connections ` + inHistory := `SELECT *, 'history' as _source FROM history.connections ` + if whereClause != "" { - query += " WHERE " + whereClause + inMem += " WHERE " + whereClause + inHistory += " WHERE " + whereClause } + source := inMem + " UNION " + inHistory + + query := `SELECT ` + selectClause + ` FROM ( ` + source + ` ) ` + query += " " + groupByClause + " " + orderByClause + " " + req.Pagination.toSQLLimitOffsetClause() return strings.TrimSpace(query), req.paramMap, nil @@ -298,8 +305,7 @@ func (req *QueryRequestPayload) generateGroupByClause(schema *orm.TableSchema) ( func (req *QueryRequestPayload) generateSelectClause() string { selectClause := "*" if len(req.selectedFields) > 0 { - selectedFields := append(req.selectedFields, "_source") - selectClause = strings.Join(selectedFields, ", ") + selectClause = strings.Join(req.selectedFields, ", ") } return selectClause From cf2b8f26b933f006ce00f9be8007b030bac589f0 Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Wed, 14 Jun 2023 10:22:32 +0200 Subject: [PATCH 03/15] Make history module optional --- netquery/database.go | 10 ++++++++-- netquery/manager.go | 4 ++-- profile/config.go | 24 ++++++++++++++++++++++++ profile/profile-layered.go | 5 +++++ profile/profile.go | 11 +++++++++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/netquery/database.go b/netquery/database.go index dab486c9..df038103 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -355,7 +355,7 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error { //nolint:un // // Save uses the database write connection instead of relying on the // connection pool. -func (db *Database) Save(ctx context.Context, conn Conn) error { +func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) error { connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig) if err != nil { return fmt.Errorf("failed to encode connection for SQL: %w", err) @@ -387,7 +387,13 @@ func (db *Database) Save(ctx context.Context, conn Conn) error { // TODO(ppacher): make sure this one can be cached to speed up inserting // and save some CPU cycles for the user - for _, dbName := range []string{"main", "history"} { + dbNames := []string{"main"} + + if enableHistory { + dbNames = append(dbNames, "history") + } + + for _, dbName := range dbNames { sql := fmt.Sprintf( `INSERT INTO %s.connections (%s) VALUES(%s) diff --git a/netquery/manager.go b/netquery/manager.go index 6599d619..bcd60618 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -25,7 +25,7 @@ type ( // insert or an update. // The ID of Conn is unique and can be trusted to never collide with other // connections of the save device. - Save(context.Context, Conn) error + Save(context.Context, Conn, bool) error } // Manager handles new and updated network.Connections feeds and persists them @@ -100,7 +100,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect log.Tracef("netquery: updating connection %s", conn.ID) - if err := mng.store.Save(ctx, *model); err != nil { + if err := mng.store.Save(ctx, *model, conn.Process().Profile().HistoryEnabled()); err != nil { log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) continue diff --git a/profile/config.go b/profile/config.go index 416de06b..3c87b009 100644 --- a/profile/config.go +++ b/profile/config.go @@ -105,6 +105,10 @@ var ( // Setting "Permanent Verdicts" at order 96. + CfgOptionEnableHistoryKey = "filter/enableHistory" + cfgOptionEnableHistory config.BoolOption + cfgOptionEnableHistoryOrder = 66 + // Setting "Enable SPN" at order 128. CfgOptionUseSPNKey = "spn/use" @@ -239,6 +243,26 @@ func registerConfiguration() error { //nolint:maintidx cfgOptionDisableAutoPermit = config.Concurrent.GetAsInt(CfgOptionDisableAutoPermitKey, int64(status.SecurityLevelsAll)) cfgIntOptions[CfgOptionDisableAutoPermitKey] = cfgOptionDisableAutoPermit + // Enable History + err = config.Register(&config.Option{ + Name: "Enable Connection History", + Key: CfgOptionEnableHistoryKey, + Description: "Whether or not to save connections to the history database", + OptType: config.OptTypeBool, + ReleaseLevel: config.ReleaseLevelExperimental, + ExpertiseLevel: config.ExpertiseLevelExpert, + DefaultValue: false, + Annotations: config.Annotations{ + config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, + config.CategoryAnnotation: "Advanced", + }, + }) + if err != nil { + return err + } + cfgOptionEnableHistory = config.Concurrent.GetAsBool(CfgOptionEnableHistoryKey, false) + cfgBoolOptions[CfgOptionEnableHistoryKey] = cfgOptionEnableHistory + rulesHelp := strings.ReplaceAll(`Rules are checked from top to bottom, stopping after the first match. They can match: - By address: "192.168.0.1" diff --git a/profile/profile-layered.go b/profile/profile-layered.go index b2f7850b..5380aca8 100644 --- a/profile/profile-layered.go +++ b/profile/profile-layered.go @@ -49,6 +49,7 @@ type LayeredProfile struct { DomainHeuristics config.BoolOption `json:"-"` UseSPN config.BoolOption `json:"-"` SPNRoutingAlgorithm config.StringOption `json:"-"` + HistoryEnabled config.BoolOption `json:"-"` } // NewLayeredProfile returns a new layered profile based on the given local profile. @@ -120,6 +121,10 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile { CfgOptionRoutingAlgorithmKey, cfgOptionRoutingAlgorithm, ) + lp.HistoryEnabled = lp.wrapBoolOption( + CfgOptionEnableHistoryKey, + cfgOptionEnableHistory, + ) lp.LayerIDs = append(lp.LayerIDs, localProfile.ScopedID()) lp.layers = append(lp.layers, localProfile) diff --git a/profile/profile.go b/profile/profile.go index 1fa12ff8..2d0eb9c4 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -136,6 +136,7 @@ type Profile struct { //nolint:maligned // not worth the effort filterListIDs []string spnUsagePolicy endpoints.Endpoints spnExitHubPolicy endpoints.Endpoints + enableHistory bool // Lifecycle Management outdated *abool.AtomicBool @@ -233,6 +234,11 @@ func (profile *Profile) parseConfig() error { } } + enableHistory, ok := profile.configPerspective.GetAsBool(CfgOptionEnableHistoryKey) + if ok { + profile.enableHistory = enableHistory + } + return lastErr } @@ -315,6 +321,11 @@ func (profile *Profile) IsOutdated() bool { return profile.outdated.IsSet() } +// HistoryEnabled returns true if connection history is enabled for the profile. +func (profile *Profile) HistoryEnabled() bool { + return profile.enableHistory +} + // GetEndpoints returns the endpoint list of the profile. This functions // requires the profile to be read locked. func (profile *Profile) GetEndpoints() endpoints.Endpoints { From dbffa8827b047c092b0a522ae822c96194c063ae Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Wed, 19 Jul 2023 11:03:11 +0200 Subject: [PATCH 04/15] Update netquery to support history module --- .gitignore | 2 + netquery/database.go | 121 ++++++++++++++++++++++++---- netquery/manager.go | 38 ++++++++- netquery/module_api.go | 76 +++++++++++++++++ netquery/orm/encoder.go | 7 +- netquery/orm/encoder_test.go | 2 +- netquery/orm/query_runner.go | 18 ++++- netquery/orm/schema_builder.go | 44 +++++++++- netquery/orm/schema_builder_test.go | 6 +- netquery/query.go | 32 +++++++- netquery/query_handler.go | 87 ++++++++++++++------ process/process.go | 4 + profile/config.go | 10 ++- 13 files changed, 391 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 7332997a..b3c96ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ _testmain.go # Custom dev scripts win_dev_* +go.work +go.work.sum diff --git a/netquery/database.go b/netquery/database.go index df038103..397a0030 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -2,18 +2,23 @@ package netquery import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" + "path" "sort" "strings" "sync" "time" + "github.com/hashicorp/go-multierror" "github.com/jackc/puddle/v2" "zombiezen.com/go/sqlite" "zombiezen.com/go/sqlite/sqlitex" + "github.com/safing/portbase/dataroot" "github.com/safing/portbase/log" "github.com/safing/portmaster/netquery/orm" "github.com/safing/portmaster/network" @@ -46,6 +51,7 @@ type ( Schema *orm.TableSchema readConnPool *puddle.Pool[*sqlite.Conn] + historyPath string l sync.Mutex writeConn *sqlite.Conn @@ -82,7 +88,9 @@ type ( Latitude float64 `sqlite:"latitude"` Longitude float64 `sqlite:"longitude"` Scope netutils.IPScope `sqlite:"scope"` - Verdict network.Verdict `sqlite:"verdict"` + WorstVerdict network.Verdict `sqlite:"worst_verdict"` + ActiveVerdict network.Verdict `sqlite:"verdict"` + FirewallVerdict network.Verdict `sqlite:"firewall_verdict"` Started time.Time `sqlite:"started,text,time"` Ended *time.Time `sqlite:"ended,text,time"` Tunneled bool `sqlite:"tunneled"` @@ -93,6 +101,8 @@ type ( Allowed *bool `sqlite:"allowed"` ProfileRevision int `sqlite:"profile_revision"` ExitNode *string `sqlite:"exit_node"` + BWIncoming uint64 `sqlite:"bw_incoming,default=0"` + BWOutgoing uint64 `sqlite:"bw_outgoing,default=0"` // TODO(ppacher): support "NOT" in search query to get rid of the following helper fields Active bool `sqlite:"active"` // could use "ended IS NOT NULL" or "ended IS NULL" @@ -108,21 +118,27 @@ type ( // (see Execute). To perform database writes use either Save() or ExecuteWrite(). // Note that write connections are serialized by the Database object before being // handed over to SQLite. -func New(path string) (*Database, error) { +func New(dbPath string) (*Database, error) { + historyParentDir := dataroot.Root().ChildDir("databases", 0o700) + if err := historyParentDir.Ensure(); err != nil { + return nil, fmt.Errorf("failed to ensure database directory exists: %w", err) + } + + historyPath := "file://" + path.Join(historyParentDir.Path, "history.db") + constructor := func(ctx context.Context) (*sqlite.Conn, error) { c, err := sqlite.OpenConn( - path, + dbPath, sqlite.OpenReadOnly, - sqlite.OpenNoMutex, //nolint:staticcheck // We like to be explicit. sqlite.OpenSharedCache, //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { - return nil, fmt.Errorf("failed to open read-only sqlite connection at %s: %w", path, err) + return nil, fmt.Errorf("failed to open read-only sqlite connection at %s: %w", dbPath, err) } - if err := sqlitex.ExecuteTransient(c, "ATTACH DATABASE 'file:///tmp/history.db?mode=ro' AS history", nil); err != nil { + if err := sqlitex.ExecuteTransient(c, "ATTACH DATABASE '"+historyPath+"?mode=ro' AS history", nil); err != nil { return nil, fmt.Errorf("failed to attach history database: %w", err) } @@ -150,23 +166,23 @@ func New(path string) (*Database, error) { } writeConn, err := sqlite.OpenConn( - path, + dbPath, sqlite.OpenCreate, sqlite.OpenReadWrite, - sqlite.OpenNoMutex, //nolint:staticcheck // We like to be explicit. sqlite.OpenWAL, sqlite.OpenSharedCache, //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { - return nil, fmt.Errorf("failed to open sqlite at %s: %w", path, err) + return nil, fmt.Errorf("failed to open sqlite at %s: %w", dbPath, err) } return &Database{ readConnPool: pool, Schema: schema, writeConn: writeConn, + historyPath: historyPath, }, nil } @@ -197,9 +213,7 @@ func (db *Database) ApplyMigrations() error { db.l.Lock() defer db.l.Unlock() - // Attach the history database - log.Errorf("attaching database history") - if err := sqlitex.ExecuteTransient(db.writeConn, "ATTACH DATABASE 'file:///tmp/history.db?mode=rwc' AS 'history';", nil); err != nil { + if err := sqlitex.ExecuteTransient(db.writeConn, "ATTACH DATABASE '"+db.historyPath+"?mode=rwc' AS 'history';", nil); err != nil { return fmt.Errorf("failed to attach history database: %w", err) } @@ -207,7 +221,7 @@ func (db *Database) ApplyMigrations() error { for _, dbName := range dbNames { // get the create-table SQL statement from the inferred schema sql := db.Schema.CreateStatement(dbName, true) - log.Errorf("creating table schema for database %q", dbName) + log.Debugf("creating table schema for database %q", dbName) // execute the SQL if err := sqlitex.ExecuteTransient(db.writeConn, sql, nil); err != nil { @@ -285,7 +299,7 @@ func (db *Database) CountRows(ctx context.Context) (int, error) { return result[0].Count, nil } -// Cleanup removes all connections that have ended before threshold. +// Cleanup removes all connections that have ended before threshold from the live database. // // NOTE(ppacher): there is no easy way to get the number of removed // rows other than counting them in a first step. Though, that's @@ -323,6 +337,18 @@ func (db *Database) Cleanup(ctx context.Context, threshold time.Time) (int, erro return result[0].Count, nil } +func (db *Database) RemoveAllHistoryData(ctx context.Context) error { + query := fmt.Sprintf("DELETE FROM %s.connections", HistoryDatabase) + return db.ExecuteWrite(ctx, query) +} + +func (db *Database) RemoveHistoryForProfile(ctx context.Context, profileID string) error { + query := fmt.Sprintf("DELETE FROM %s.connections WHERE profile = :profile", HistoryDatabase) + return db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{ + ":profile": profileID, + })) +} + // dumpTo is a simple helper method that dumps all rows stored in the SQLite database // as JSON to w. // Any error aborts dumping rows and is returned. @@ -350,13 +376,74 @@ func (db *Database) dumpTo(ctx context.Context, w io.Writer) error { //nolint:un return enc.Encode(conns) } +// MarkAllHistoryConnectionsEnded marks all connections in the history database as ended. +func (db *Database) MarkAllHistoryConnectionsEnded(ctx context.Context) error { + query := fmt.Sprintf("UPDATE %s.connections SET active = FALSE, ended = :ended WHERE active = TRUE", HistoryDatabase) + + if err := db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{ + ":ended": time.Now().Format(orm.SqliteTimeFormat), + })); err != nil { + return err + } + + return nil +} + +func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, incoming *uint64, outgoing *uint64) error { + data := connID + "-" + processKey + hash := sha256.Sum256([]byte(data)) + dbConnId := hex.EncodeToString(hash[:]) + + params := map[string]any{ + ":id": dbConnId, + } + + parts := []string{} + if incoming != nil { + parts = append(parts, "bw_incoming = :bw_incoming") + params[":bw_incoming"] = *incoming + } + + if outgoing != nil { + parts = append(parts, "bw_outgoing = :bw_outgoing") + params[":bw_outgoing"] = *outgoing + } + + updateSet := strings.Join(parts, ", ") + + updateStmts := []string{ + fmt.Sprintf(`UPDATE %s.connections SET %s WHERE id = :id`, LiveDatabase, updateSet), + } + + if enableHistory { + updateStmts = append(updateStmts, + fmt.Sprintf(`UPDATE %s.connections SET %s WHERE id = :id`, HistoryDatabase, updateSet), + ) + } + + merr := new(multierror.Error) + for _, stmt := range updateStmts { + if err := db.ExecuteWrite(ctx, stmt, orm.WithNamedArgs(params)); err != nil { + merr.Errors = append(merr.Errors, err) + } + } + + return merr.ErrorOrNil() +} + // Save inserts the connection conn into the SQLite database. If conn // already exists the table row is updated instead. // // Save uses the database write connection instead of relying on the // connection pool. func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) error { - connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig) + // convert the connection to a param map where each key is already translated + // to the sql column name. We also skip bw_incoming and bw_outgoing since those + // will be updated independenly from the connection object. + connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig, []string{ + "bw_incoming", + "bw_outgoing", + }) if err != nil { return fmt.Errorf("failed to encode connection for SQL: %w", err) } @@ -387,10 +474,10 @@ func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) err // TODO(ppacher): make sure this one can be cached to speed up inserting // and save some CPU cycles for the user - dbNames := []string{"main"} + dbNames := []DatabaseName{LiveDatabase} if enableHistory { - dbNames = append(dbNames, "history") + dbNames = append(dbNames, HistoryDatabase) } for _, dbName := range dbNames { diff --git a/netquery/manager.go b/netquery/manager.go index bcd60618..531063b8 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -13,6 +13,8 @@ import ( "github.com/safing/portbase/log" "github.com/safing/portbase/runtime" "github.com/safing/portmaster/network" + "github.com/safing/spn/access" + "github.com/safing/spn/access/account" ) type ( @@ -26,6 +28,21 @@ type ( // The ID of Conn is unique and can be trusted to never collide with other // connections of the save device. Save(context.Context, Conn, bool) error + + // MarkAllHistoryConnectionsEnded marks all active connections in the history + // database as ended NOW. + MarkAllHistoryConnectionsEnded(context.Context) error + + // RemoveHistoryForProfile removes all connections from the history database + // for a given profile ID (source/id) + RemoveHistoryForProfile(context.Context, string) error + + // RemoveAllHistoryData removes all connections from the history database. + RemoveAllHistoryData(context.Context) error + + // UpdateBandwidth updates bandwith data for the connection and optionally also writes + // the bandwidth data to the history database. + UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, incoming *uint64, outgoing *uint64) error } // Manager handles new and updated network.Connections feeds and persists them @@ -100,7 +117,20 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect log.Tracef("netquery: updating connection %s", conn.ID) - if err := mng.store.Save(ctx, *model, conn.Process().Profile().HistoryEnabled()); err != nil { + // check if we should persist the connection in the history database. + // Also make sure the current SPN User/subscription allows use of the history. + historyEnabled := conn.Process().Profile().HistoryEnabled() + if historyEnabled { + user, err := access.GetUser() + if err != nil { + // there was an error so disable history + historyEnabled = false + } else if !user.MayUse(account.FeatureHistory) { + historyEnabled = false + } + } + + if err := mng.store.Save(ctx, *model, historyEnabled); err != nil { log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) continue @@ -158,7 +188,9 @@ func convertConnection(conn *network.Connection) (*Conn, error) { IPProtocol: conn.IPProtocol, LocalIP: conn.LocalIP.String(), LocalPort: conn.LocalPort, - Verdict: conn.Verdict.Firewall, // TODO: Expose both Worst and Firewall verdicts. + FirewallVerdict: conn.Verdict.Firewall, + ActiveVerdict: conn.Verdict.Active, + WorstVerdict: conn.Verdict.Worst, Started: time.Unix(conn.Started, 0), Tunneled: conn.Tunneled, Encrypted: conn.Encrypted, @@ -250,7 +282,7 @@ func convertConnection(conn *network.Connection) (*Conn, error) { } func genConnID(conn *network.Connection) string { - data := conn.ID + "-" + time.Unix(conn.Started, 0).String() + data := conn.ID + "-" + conn.Process().GetID() hash := sha256.Sum256([]byte(data)) return hex.EncodeToString(hash[:]) } diff --git a/netquery/module_api.go b/netquery/module_api.go index 4cb02462..3746fdf4 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -2,15 +2,19 @@ package netquery import ( "context" + "encoding/json" "fmt" + "net/http" "time" + "github.com/hashicorp/go-multierror" "github.com/safing/portbase/api" "github.com/safing/portbase/config" "github.com/safing/portbase/database" "github.com/safing/portbase/database/query" "github.com/safing/portbase/log" "github.com/safing/portbase/modules" + "github.com/safing/portbase/modules/subsystems" "github.com/safing/portbase/runtime" "github.com/safing/portmaster/network" ) @@ -35,6 +39,15 @@ func init() { "network", "database", ) + + subsystems.Register( + "history", + "Network History", + "Keep Network History Data", + m.Module, + "config:history/", + nil, + ) } func (m *module) prepare() error { @@ -92,6 +105,58 @@ func (m *module) prepare() error { return fmt.Errorf("failed to register API endpoint: %w", err) } + if err := api.RegisterEndpoint(api.Endpoint{ + Path: "netquery/history/clear", + MimeType: "application/json", + Read: api.PermitUser, + Write: api.PermitUser, + BelongsTo: m.Module, + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + var body struct { + ProfileIDs []string `json:"profileIDs"` + } + + defer r.Body.Close() + + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + + if err := dec.Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if len(body.ProfileIDs) == 0 { + if err := m.mng.store.RemoveAllHistoryData(r.Context()); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + } else { + merr := new(multierror.Error) + for _, profileID := range body.ProfileIDs { + if err := m.mng.store.RemoveHistoryForProfile(r.Context(), profileID); err != nil { + merr.Errors = append(merr.Errors, fmt.Errorf("failed to clear history for %q: %w", profileID, err)) + } else { + log.Infof("netquery: successfully cleared history for %s", profileID) + } + } + + if err := merr.ErrorOrNil(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + } + + w.WriteHeader(http.StatusNoContent) + }, + Name: "Remove connections from profile history", + Description: "Remove all connections from the history database for one or more profiles", + }); err != nil { + return fmt.Errorf("failed to register API endpoint: %w", err) + } + return nil } @@ -163,5 +228,16 @@ func (m *module) start() error { } func (m *module) stop() error { + // we don't use m.Module.Ctx here because it is already cancelled when stop is called. + // just give the clean up 1 minute to happen and abort otherwise. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if err := m.mng.store.MarkAllHistoryConnectionsEnded(ctx); err != nil { + // handle the error by just logging it. There's not much we can do here + // and returning an error to the module system doesn't help much as well... + log.Errorf("failed to mark connections in history database as eded: %w", err) + } + return nil } diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go index 7961f088..ef86b842 100644 --- a/netquery/orm/encoder.go +++ b/netquery/orm/encoder.go @@ -6,6 +6,7 @@ import ( "reflect" "time" + "golang.org/x/exp/slices" "zombiezen.com/go/sqlite" ) @@ -22,7 +23,7 @@ type ( // ToParamMap returns a map that contains the sqlite compatible value of each struct field of // r using the sqlite column name as a map key. It either uses the name of the // exported struct field or the value of the "sqlite" tag. -func ToParamMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig) (map[string]interface{}, error) { +func ToParamMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig, skipFields []string) (map[string]interface{}, error) { // make sure we work on a struct type val := reflect.Indirect(reflect.ValueOf(r)) if val.Kind() != reflect.Struct { @@ -45,6 +46,10 @@ func ToParamMap(ctx context.Context, r interface{}, keyPrefix string, cfg Encode return nil, fmt.Errorf("failed to get column definition for %s: %w", fieldType.Name, err) } + if slices.Contains(skipFields, colDef.Name) { + continue + } + x, found, err := runEncodeHooks( colDef, fieldType.Type, diff --git a/netquery/orm/encoder_test.go b/netquery/orm/encoder_test.go index e5142962..d0d3c039 100644 --- a/netquery/orm/encoder_test.go +++ b/netquery/orm/encoder_test.go @@ -119,7 +119,7 @@ func TestEncodeAsMap(t *testing.T) { //nolint:tparallel for idx := range cases { //nolint:paralleltest c := cases[idx] t.Run(c.Desc, func(t *testing.T) { - res, err := ToParamMap(ctx, c.Input, "", DefaultEncodeConfig) + res, err := ToParamMap(ctx, c.Input, "", DefaultEncodeConfig, nil) assert.NoError(t, err) assert.Equal(t, c.Expected, res) }) diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go index 55bafe30..f59cca79 100644 --- a/netquery/orm/query_runner.go +++ b/netquery/orm/query_runner.go @@ -143,7 +143,23 @@ func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...Q currentField := reflect.New(valElemType) if err := DecodeStmt(ctx, &args.Schema, stmt, currentField.Interface(), args.DecodeConfig); err != nil { - return err + resultDump := make(map[string]any) + + for colIdx := 0; colIdx < stmt.ColumnCount(); colIdx++ { + name := stmt.ColumnName(colIdx) + + switch stmt.ColumnType(colIdx) { + case sqlite.TypeText: + resultDump[name] = stmt.ColumnText(colIdx) + case sqlite.TypeFloat: + resultDump[name] = stmt.ColumnFloat(colIdx) + case sqlite.TypeInteger: + resultDump[name] = stmt.ColumnInt(colIdx) + case sqlite.TypeNull: + resultDump[name] = "" + } + } + return fmt.Errorf("%w: %+v", err, resultDump) } sliceVal = reflect.Append(sliceVal, reflect.Indirect(currentField)) diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go index e9e46874..080c5003 100644 --- a/netquery/orm/schema_builder.go +++ b/netquery/orm/schema_builder.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/safing/portbase/log" "zombiezen.com/go/sqlite" ) @@ -25,6 +26,7 @@ var ( TagTypePrefixVarchar = "varchar" TagTypeBlob = "blob" TagTypeFloat = "float" + TagTypePrefixDefault = "default=" ) var sqlTypeMap = map[sqlite.ColumnType]string{ @@ -52,6 +54,7 @@ type ( AutoIncrement bool UnixNano bool IsTime bool + Default any } ) @@ -105,6 +108,21 @@ func (def ColumnDef) AsSQL() string { if def.AutoIncrement { sql += " AUTOINCREMENT" } + if def.Default != nil { + sql += " DEFAULT " + switch def.Type { + case sqlite.TypeFloat: + sql += strconv.FormatFloat(def.Default.(float64), 'b', 0, 64) + case sqlite.TypeInteger: + sql += strconv.FormatInt(def.Default.(int64), 10) + case sqlite.TypeText: + sql += fmt.Sprintf("%q", def.Default.(string)) + default: + log.Errorf("unsupported default value: %q %q", def.Type, def.Default) + sql = strings.TrimSuffix(sql, " DEFAULT ") + } + sql += " " + } if !def.Nullable { sql += " NOT NULL" } @@ -160,7 +178,7 @@ func getColumnDef(fieldType reflect.StructField) (*ColumnDef, error) { kind := normalizeKind(ft.Kind()) switch kind { //nolint:exhaustive - case reflect.Int: + case reflect.Int, reflect.Uint: def.Type = sqlite.TypeInteger case reflect.Float64: @@ -237,6 +255,30 @@ func applyStructFieldTag(fieldType reflect.StructField, def *ColumnDef) error { def.Length = int(length) } + if strings.HasPrefix(k, TagTypePrefixDefault) { + defaultValue := strings.TrimPrefix(k, TagTypePrefixDefault) + switch def.Type { + case sqlite.TypeFloat: + fv, err := strconv.ParseFloat(defaultValue, 64) + if err != nil { + return fmt.Errorf("failed to parse default value as float %q: %w", defaultValue, err) + } + def.Default = fv + case sqlite.TypeInteger: + fv, err := strconv.ParseInt(defaultValue, 10, 0) + if err != nil { + return fmt.Errorf("failed to parse default value as int %q: %w", defaultValue, err) + } + def.Default = fv + case sqlite.TypeText: + def.Default = defaultValue + case sqlite.TypeBlob: + return fmt.Errorf("default values for TypeBlob not yet supported") + default: + return fmt.Errorf("failed to apply default value for unknown sqlite column type %s", def.Type) + } + } + } } } diff --git a/netquery/orm/schema_builder_test.go b/netquery/orm/schema_builder_test.go index 734da981..fdd43ec7 100644 --- a/netquery/orm/schema_builder_test.go +++ b/netquery/orm/schema_builder_test.go @@ -22,14 +22,14 @@ func TestSchemaBuilder(t *testing.T) { Int *int `sqlite:",not-null"` Float interface{} `sqlite:",float,nullable"` }{}, - `CREATE TABLE Simple ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, text TEXT, Int INTEGER NOT NULL, Float REAL );`, + `CREATE TABLE main.Simple ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, text TEXT, Int INTEGER NOT NULL, Float REAL );`, }, { "Varchar", struct { S string `sqlite:",varchar(10)"` }{}, - `CREATE TABLE Varchar ( S VARCHAR(10) NOT NULL );`, + `CREATE TABLE main.Varchar ( S VARCHAR(10) NOT NULL );`, }, } @@ -38,6 +38,6 @@ func TestSchemaBuilder(t *testing.T) { res, err := GenerateTableSchema(c.Name, c.Model) assert.NoError(t, err) - assert.Equal(t, c.ExpectedSQL, res.CreateStatement(false)) + assert.Equal(t, c.ExpectedSQL, res.CreateStatement("main", false)) } } diff --git a/netquery/query.go b/netquery/query.go index 83dbc217..264f0bd7 100644 --- a/netquery/query.go +++ b/netquery/query.go @@ -14,6 +14,13 @@ import ( "github.com/safing/portmaster/netquery/orm" ) +type DatabaseName string + +const ( + LiveDatabase = DatabaseName("main") + HistoryDatabase = DatabaseName("history") +) + // Collection of Query and Matcher types. // NOTE: whenever adding support for new operators make sure // to update UnmarshalJSON as well. @@ -48,11 +55,19 @@ type ( Distinct bool `json:"distinct"` } + Min struct { + Condition *Query `json:"condition,omitempty"` + Field string `json:"field"` + As string `json:"as"` + Distinct bool `json:"distinct"` + } + Select struct { Field string `json:"field"` Count *Count `json:"$count,omitempty"` Sum *Sum `json:"$sum,omitempty"` - Distinct *string `json:"$distinct"` + Min *Min `json:"$min,omitempty"` + Distinct *string `json:"$distinct,omitempty"` } Selects []Select @@ -68,6 +83,9 @@ type ( OrderBy OrderBys `json:"orderBy"` GroupBy []string `json:"groupBy"` TextSearch *TextSearch `json:"textSearch"` + // A list of databases to query. If left empty, + // both, the LiveDatabase and the HistoryDatabase are queried + Databases []DatabaseName `json:"databases"` Pagination @@ -457,6 +475,7 @@ func (sel *Select) UnmarshalJSON(blob []byte) error { Field string `json:"field"` Count *Count `json:"$count"` Sum *Sum `json:"$sum"` + Min *Min `json:"$min"` Distinct *string `json:"$distinct"` } @@ -468,12 +487,23 @@ func (sel *Select) UnmarshalJSON(blob []byte) error { sel.Field = res.Field sel.Distinct = res.Distinct sel.Sum = res.Sum + sel.Min = res.Min if sel.Count != nil && sel.Count.As != "" { if !charOnlyRegexp.MatchString(sel.Count.As) { return fmt.Errorf("invalid characters in $count.as, value must match [a-zA-Z]+") } } + if sel.Sum != nil && sel.Sum.As != "" { + if !charOnlyRegexp.MatchString(sel.Sum.As) { + return fmt.Errorf("invalid characters in $sum.as, value must match [a-zA-Z]+") + } + } + if sel.Min != nil && sel.Min.As != "" { + if !charOnlyRegexp.MatchString(sel.Min.As) { + return fmt.Errorf("invalid characters in $min.as, value must match [a-zA-Z]+") + } + } return nil } diff --git a/netquery/query_handler.go b/netquery/query_handler.go index 141f7dd5..3c6bb453 100644 --- a/netquery/query_handler.go +++ b/netquery/query_handler.go @@ -14,6 +14,7 @@ import ( "github.com/safing/portbase/log" "github.com/safing/portmaster/netquery/orm" + "golang.org/x/exp/slices" ) var charOnlyRegexp = regexp.MustCompile("[a-zA-Z]+") @@ -152,13 +153,7 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab return "", nil, fmt.Errorf("generating where clause: %w", err) } - if req.paramMap == nil { - req.paramMap = make(map[string]interface{}) - } - - for key, val := range paramMap { - req.paramMap[key] = val - } + req.mergeParams(paramMap) if req.TextSearch != nil { textClause, textParams, err := req.TextSearch.toSQLConditionClause(ctx, schema, "", orm.DefaultEncodeConfig) @@ -173,9 +168,7 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab whereClause += textClause - for key, val := range textParams { - req.paramMap[key] = val - } + req.mergeParams(textParams) } } @@ -190,15 +183,21 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab } selectClause := req.generateSelectClause() - inMem := `SELECT *, 'live' as _source FROM main.connections ` - inHistory := `SELECT *, 'history' as _source FROM history.connections ` if whereClause != "" { - inMem += " WHERE " + whereClause - inHistory += " WHERE " + whereClause + whereClause = "WHERE " + whereClause } - source := inMem + " UNION " + inHistory + if len(req.Databases) == 0 { + req.Databases = []DatabaseName{LiveDatabase, HistoryDatabase} + } + + sources := make([]string, len(req.Databases)) + for idx, db := range req.Databases { + sources[idx] = fmt.Sprintf("SELECT * FROM %s.connections %s", db, whereClause) + } + + source := strings.Join(sources, " UNION ") query := `SELECT ` + selectClause + ` FROM ( ` + source + ` ) ` @@ -210,6 +209,7 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schema *orm.TableSchema) error { for idx, s := range req.Select { var field string + switch { case s.Count != nil: field = s.Count.Field @@ -218,6 +218,12 @@ func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schem case s.Sum != nil: // field is not used in case of $sum field = "*" + case s.Min != nil: + if s.Min.Field != "" { + field = s.Min.Field + } else { + field = "*" + } default: field = s.Field } @@ -258,13 +264,40 @@ func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schem return fmt.Errorf("in $sum: %w", err) } - req.paramMap = params + req.mergeParams(params) req.selectedFields = append( req.selectedFields, fmt.Sprintf("SUM(%s) AS %s", clause, s.Sum.As), ) req.whitelistedFields = append(req.whitelistedFields, s.Sum.As) + case s.Min != nil: + if s.Min.As == "" { + return fmt.Errorf("missing 'as' for $min") + } + + var ( + clause string + params map[string]any + ) + + if s.Min.Field != "" { + clause = field + } else { + var err error + clause, params, err = s.Min.Condition.toSQLWhereClause(ctx, fmt.Sprintf("sel%d", idx), schema, orm.DefaultEncodeConfig) + if err != nil { + return fmt.Errorf("in $min: %w", err) + } + } + + req.mergeParams(params) + req.selectedFields = append( + req.selectedFields, + fmt.Sprintf("MIN(%s) AS %s", clause, s.Min.As), + ) + req.whitelistedFields = append(req.whitelistedFields, s.Min.As) + case s.Distinct != nil: req.selectedFields = append(req.selectedFields, fmt.Sprintf("DISTINCT %s", colName)) req.whitelistedFields = append(req.whitelistedFields, colName) @@ -277,6 +310,16 @@ func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schem return nil } +func (req *QueryRequestPayload) mergeParams(params map[string]any) { + if req.paramMap == nil { + req.paramMap = make(map[string]any) + } + + for key, value := range params { + req.paramMap[key] = value + } +} + func (req *QueryRequestPayload) generateGroupByClause(schema *orm.TableSchema) (string, error) { if len(req.GroupBy) == 0 { return "", nil @@ -339,16 +382,12 @@ func (req *QueryRequestPayload) validateColumnName(schema *orm.TableSchema, fiel return colDef.Name, nil } - for _, selected := range req.whitelistedFields { - if field == selected { - return field, nil - } + if slices.Contains(req.whitelistedFields, field) { + return field, nil } - for _, selected := range req.selectedFields { - if field == selected { - return field, nil - } + if slices.Contains(req.selectedFields, field) { + return field, nil } return "", fmt.Errorf("column name %q not allowed", field) diff --git a/process/process.go b/process/process.go index 3f2779f9..99c281cc 100644 --- a/process/process.go +++ b/process/process.go @@ -313,6 +313,10 @@ func loadProcess(ctx context.Context, key string, pInfo *processInfo.Process) (* return process, nil } +func (p *Process) GetID() string { + return p.processKey +} + // Builds a unique identifier for a processes. func getProcessKey(pid int32, createdTime int64) string { return fmt.Sprintf("%d-%d", pid, createdTime) diff --git a/profile/config.go b/profile/config.go index 3c87b009..51fd731b 100644 --- a/profile/config.go +++ b/profile/config.go @@ -6,6 +6,7 @@ import ( "github.com/safing/portbase/config" "github.com/safing/portmaster/profile/endpoints" "github.com/safing/portmaster/status" + "github.com/safing/spn/access/account" "github.com/safing/spn/navigator" ) @@ -105,7 +106,7 @@ var ( // Setting "Permanent Verdicts" at order 96. - CfgOptionEnableHistoryKey = "filter/enableHistory" + CfgOptionEnableHistoryKey = "history/enabled" cfgOptionEnableHistory config.BoolOption cfgOptionEnableHistoryOrder = 66 @@ -249,12 +250,13 @@ func registerConfiguration() error { //nolint:maintidx Key: CfgOptionEnableHistoryKey, Description: "Whether or not to save connections to the history database", OptType: config.OptTypeBool, - ReleaseLevel: config.ReleaseLevelExperimental, + ReleaseLevel: config.ReleaseLevelStable, ExpertiseLevel: config.ExpertiseLevelExpert, DefaultValue: false, Annotations: config.Annotations{ - config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, - config.CategoryAnnotation: "Advanced", + config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, + config.CategoryAnnotation: "History", + config.SettingRequiresFeaturePlan: account.FeatureHistory, }, }) if err != nil { From b7fd1fc76aef71925c2261f4531cf02fcfce55bd Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 10:56:50 +0200 Subject: [PATCH 05/15] Update config annotation --- profile/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profile/config.go b/profile/config.go index 51fd731b..424b459a 100644 --- a/profile/config.go +++ b/profile/config.go @@ -254,9 +254,9 @@ func registerConfiguration() error { //nolint:maintidx ExpertiseLevel: config.ExpertiseLevelExpert, DefaultValue: false, Annotations: config.Annotations{ - config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, - config.CategoryAnnotation: "History", - config.SettingRequiresFeaturePlan: account.FeatureHistory, + config.DisplayOrderAnnotation: cfgOptionEnableHistoryOrder, + config.CategoryAnnotation: "History", + config.RequiresFeatureID: account.FeatureHistory, }, }) if err != nil { From 5dcb6b268f8a4055208c3a40eb4a8d8bfc00da39 Mon Sep 17 00:00:00 2001 From: Patrick Pacher Date: Fri, 21 Jul 2023 11:38:05 +0200 Subject: [PATCH 06/15] Persist bandwidth data in netquery DBs when enabled --- firewall/module.go | 2 +- firewall/packet_handler.go | 18 +++++++++++++++--- netquery/database.go | 18 +++++++++--------- netquery/manager.go | 17 +---------------- netquery/module_api.go | 36 ++++++++++++++++++++---------------- network/connection.go | 23 +++++++++++++++++++++++ 6 files changed, 69 insertions(+), 45 deletions(-) diff --git a/firewall/module.go b/firewall/module.go index 345316c0..dd4dcbaa 100644 --- a/firewall/module.go +++ b/firewall/module.go @@ -14,7 +14,7 @@ import ( var module *modules.Module func init() { - module = modules.Register("filter", prep, start, stop, "core", "interception", "intel") + module = modules.Register("filter", prep, start, stop, "core", "interception", "intel", "netquery") subsystems.Register( "filter", "Privacy Filter", diff --git a/firewall/packet_handler.go b/firewall/packet_handler.go index 4fc783ba..133e7dba 100644 --- a/firewall/packet_handler.go +++ b/firewall/packet_handler.go @@ -18,6 +18,7 @@ import ( "github.com/safing/portmaster/firewall/inspection" "github.com/safing/portmaster/firewall/interception" "github.com/safing/portmaster/netenv" + "github.com/safing/portmaster/netquery" "github.com/safing/portmaster/network" "github.com/safing/portmaster/network/netutils" "github.com/safing/portmaster/network/packet" @@ -616,7 +617,7 @@ func bandwidthUpdateHandler(ctx context.Context) error { return nil case bwUpdate := <-interception.BandwidthUpdates: if bwUpdate != nil { - updateBandwidth(bwUpdate) + updateBandwidth(ctx, bwUpdate) // DEBUG: // log.Debugf("filter: bandwidth update: %s", bwUpdate) } else { @@ -626,7 +627,7 @@ func bandwidthUpdateHandler(ctx context.Context) error { } } -func updateBandwidth(bwUpdate *packet.BandwidthUpdate) { +func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { // Check if update makes sense. if bwUpdate.RecvBytes == 0 && bwUpdate.SentBytes == 0 { return @@ -657,7 +658,18 @@ func updateBandwidth(bwUpdate *packet.BandwidthUpdate) { log.Warningf("filter: unsupported bandwidth update method: %d", bwUpdate.Method) } - // TODO: Send update. + if netquery.DefaultModule != nil && conn.BandwidthEnabled { + if err := netquery.DefaultModule.Store.UpdateBandwidth( + ctx, + conn.HistoryEnabled, + conn.Process().GetID(), + conn.ID, + &conn.RecvBytes, + &conn.SentBytes, + ); err != nil { + log.Errorf("firewall: failed to persist bandwidth data: %s", err) + } + } } func statLogger(ctx context.Context) error { diff --git a/netquery/database.go b/netquery/database.go index 397a0030..a5ce6c01 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -101,8 +101,8 @@ type ( Allowed *bool `sqlite:"allowed"` ProfileRevision int `sqlite:"profile_revision"` ExitNode *string `sqlite:"exit_node"` - BWIncoming uint64 `sqlite:"bw_incoming,default=0"` - BWOutgoing uint64 `sqlite:"bw_outgoing,default=0"` + BytesReceived uint64 `sqlite:"bytes_received,default=0"` + BytesSent uint64 `sqlite:"bytes_sent,default=0"` // TODO(ppacher): support "NOT" in search query to get rid of the following helper fields Active bool `sqlite:"active"` // could use "ended IS NOT NULL" or "ended IS NULL" @@ -400,13 +400,13 @@ func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, pro parts := []string{} if incoming != nil { - parts = append(parts, "bw_incoming = :bw_incoming") - params[":bw_incoming"] = *incoming + parts = append(parts, "bytes_received = :bytes_received") + params[":bytes_received"] = *incoming } if outgoing != nil { - parts = append(parts, "bw_outgoing = :bw_outgoing") - params[":bw_outgoing"] = *outgoing + parts = append(parts, "bytes_sent = :bytes_sent") + params[":bytes_sent"] = *outgoing } updateSet := strings.Join(parts, ", ") @@ -438,11 +438,11 @@ func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, pro // connection pool. func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) error { // convert the connection to a param map where each key is already translated - // to the sql column name. We also skip bw_incoming and bw_outgoing since those + // to the sql column name. We also skip bytes_received and bytes_sent since those // will be updated independenly from the connection object. connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig, []string{ - "bw_incoming", - "bw_outgoing", + "bytes_received", + "bytes_sent", }) if err != nil { return fmt.Errorf("failed to encode connection for SQL: %w", err) diff --git a/netquery/manager.go b/netquery/manager.go index 531063b8..e34baa9c 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -13,8 +13,6 @@ import ( "github.com/safing/portbase/log" "github.com/safing/portbase/runtime" "github.com/safing/portmaster/network" - "github.com/safing/spn/access" - "github.com/safing/spn/access/account" ) type ( @@ -117,20 +115,7 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect log.Tracef("netquery: updating connection %s", conn.ID) - // check if we should persist the connection in the history database. - // Also make sure the current SPN User/subscription allows use of the history. - historyEnabled := conn.Process().Profile().HistoryEnabled() - if historyEnabled { - user, err := access.GetUser() - if err != nil { - // there was an error so disable history - historyEnabled = false - } else if !user.MayUse(account.FeatureHistory) { - historyEnabled = false - } - } - - if err := mng.store.Save(ctx, *model, historyEnabled); err != nil { + if err := mng.store.Save(ctx, *model, conn.HistoryEnabled); err != nil { log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) continue diff --git a/netquery/module_api.go b/netquery/module_api.go index 3746fdf4..f102ec65 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -19,22 +19,26 @@ import ( "github.com/safing/portmaster/network" ) +var DefaultModule *module + type module struct { *modules.Module - db *database.Interface - sqlStore *Database - mng *Manager - feed chan *network.Connection + Store *Database + + db *database.Interface + mng *Manager + feed chan *network.Connection } func init() { - m := new(module) - m.Module = modules.Register( + DefaultModule = new(module) + + DefaultModule.Module = modules.Register( "netquery", - m.prepare, - m.start, - m.stop, + DefaultModule.prepare, + DefaultModule.start, + DefaultModule.stop, "api", "network", "database", @@ -44,7 +48,7 @@ func init() { "history", "Network History", "Keep Network History Data", - m.Module, + DefaultModule.Module, "config:history/", nil, ) @@ -58,12 +62,12 @@ func (m *module) prepare() error { Internal: true, }) - m.sqlStore, err = NewInMemory() + m.Store, err = NewInMemory() if err != nil { return fmt.Errorf("failed to create in-memory database: %w", err) } - m.mng, err = NewManager(m.sqlStore, "netquery/data/", runtime.DefaultRegistry) + m.mng, err = NewManager(m.Store, "netquery/data/", runtime.DefaultRegistry) if err != nil { return fmt.Errorf("failed to create manager: %w", err) } @@ -71,12 +75,12 @@ func (m *module) prepare() error { m.feed = make(chan *network.Connection, 1000) queryHander := &QueryHandler{ - Database: m.sqlStore, + Database: m.Store, IsDevMode: config.Concurrent.GetAsBool(config.CfgDevModeKey, false), } chartHandler := &ChartHandler{ - Database: m.sqlStore, + Database: m.Store, } if err := api.RegisterEndpoint(api.Endpoint{ @@ -204,7 +208,7 @@ func (m *module) start() error { return nil case <-time.After(10 * time.Second): threshold := time.Now().Add(-network.DeleteConnsAfterEndedThreshold) - count, err := m.sqlStore.Cleanup(ctx, threshold) + count, err := m.Store.Cleanup(ctx, threshold) if err != nil { log.Errorf("netquery: failed to count number of rows in memory: %s", err) } else { @@ -218,7 +222,7 @@ func (m *module) start() error { // the runtime database. // Only expose in development mode. if config.GetAsBool(config.CfgDevModeKey, false)() { - _, err := NewRuntimeQueryRunner(m.sqlStore, "netquery/query/", runtime.DefaultRegistry) + _, err := NewRuntimeQueryRunner(m.Store, "netquery/query/", runtime.DefaultRegistry) if err != nil { return fmt.Errorf("failed to set up runtime SQL query runner: %w", err) } diff --git a/network/connection.go b/network/connection.go index 63182ca2..3db4949d 100644 --- a/network/connection.go +++ b/network/connection.go @@ -19,6 +19,8 @@ import ( "github.com/safing/portmaster/process" _ "github.com/safing/portmaster/process/tags" "github.com/safing/portmaster/resolver" + "github.com/safing/spn/access" + "github.com/safing/spn/access/account" "github.com/safing/spn/navigator" ) @@ -218,6 +220,13 @@ type Connection struct { //nolint:maligned // TODO: fix alignment // addedToMetrics signifies if the connection has already been counted in // the metrics. addedToMetrics bool + + // HistoryEnabled is set to true when the connection should be persisted + // in the history database. + HistoryEnabled bool + // BanwidthEnabled is set to true if connection bandwidth data should be persisted + // in netquery. + BandwidthEnabled bool } // Reason holds information justifying a verdict, as well as additional @@ -420,7 +429,21 @@ func (conn *Connection) GatherConnectionInfo(pkt packet.Packet) (err error) { // Inherit internal status of profile. if localProfile := conn.process.Profile().LocalProfile(); localProfile != nil { conn.Internal = localProfile.Internal + + // check if we should persist the connection in the history database. + // Also make sure the current SPN User/subscription allows use of the history. + user, err := access.GetUser() + if err == nil { + if user.MayUse(account.FeatureHistory) { + conn.HistoryEnabled = localProfile.HistoryEnabled() + } + + if user.MayUse(account.FeatureBWVis) { + conn.BandwidthEnabled = true + } + } } + } else { conn.process = nil if pkt.InfoOnly() { From 49adef242eb0c07bfb5cd493f779c7e39abaca09 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:03:26 +0200 Subject: [PATCH 07/15] Update links in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c57104c1..3ee3ae07 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,12 @@ Portmaster is a privacy suite for your desktop OS. All details and guides live in the dedicated [wiki](https://wiki.safing.io/) -- [Getting Started](https://wiki.safing.io/en/Portmaster/App/GettingStarted) +- [Getting Started](https://wiki.safing.io/en/Portmaster/App) - Install - [on Windows](https://wiki.safing.io/en/Portmaster/Install/Windows) - [on Linux](https://wiki.safing.io/en/Portmaster/Install/Linux) - [Contribute](https://wiki.safing.io/en/Contribute) - [VPN Compatibility](https://wiki.safing.io/en/Portmaster/App/Compatibility#vpn-compatibly) - [Software Compatibility](https://wiki.safing.io/en/Portmaster/App/Compatibility) -- [Architecture](https://wiki.safing.io/en/Portmaster/Architecture/Overview) +- [Architecture](https://wiki.safing.io/en/Portmaster/Architecture) From e70fd9abd7734c9a7f57f037c1ec71e69240a0b0 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:04:02 +0200 Subject: [PATCH 08/15] Reduce noisy logging --- firewall/interception/ebpf/connection_listener/worker.go | 3 ++- firewall/interception/nfq/nfq.go | 3 ++- firewall/interception/nfq/packet.go | 9 +++++++-- netquery/manager.go | 3 ++- network/clean.go | 3 ++- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/firewall/interception/ebpf/connection_listener/worker.go b/firewall/interception/ebpf/connection_listener/worker.go index d8aced12..4798e389 100644 --- a/firewall/interception/ebpf/connection_listener/worker.go +++ b/firewall/interception/ebpf/connection_listener/worker.go @@ -114,7 +114,8 @@ func ConnectionListenerWorker(ctx context.Context, packets chan packet.Packet) e PID: int(event.Pid), }) if isEventValid(event) { - log.Debugf("ebpf: received valid connect event: PID: %d Conn: %s", pkt.Info().PID, pkt) + // DEBUG: + // log.Debugf("ebpf: received valid connect event: PID: %d Conn: %s", pkt.Info().PID, pkt) packets <- pkt } else { log.Warningf("ebpf: received invalid connect event: PID: %d Conn: %s", pkt.Info().PID, pkt) diff --git a/firewall/interception/nfq/nfq.go b/firewall/interception/nfq/nfq.go index 585ba96e..184e15f9 100644 --- a/firewall/interception/nfq/nfq.go +++ b/firewall/interception/nfq/nfq.go @@ -196,7 +196,8 @@ func (q *Queue) packetHandler(ctx context.Context) func(nfqueue.Attribute) int { select { case q.packets <- pkt: - log.Tracef("nfqueue: queued packet %s (%s -> %s) after %s", pkt.ID(), pkt.Info().Src, pkt.Info().Dst, time.Since(pkt.Info().SeenAt)) + // DEBUG: + // log.Tracef("nfqueue: queued packet %s (%s -> %s) after %s", pkt.ID(), pkt.Info().Src, pkt.Info().Dst, time.Since(pkt.Info().SeenAt)) case <-ctx.Done(): return 0 case <-time.After(time.Second): diff --git a/firewall/interception/nfq/packet.go b/firewall/interception/nfq/packet.go index 6dd42186..8baeff5b 100644 --- a/firewall/interception/nfq/packet.go +++ b/firewall/interception/nfq/packet.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "sync/atomic" - "time" "github.com/florianl/go-nfqueue" "github.com/tevino/abool" @@ -117,7 +116,13 @@ func (pkt *packet) setMark(mark int) error { } break } - log.Tracer(pkt.Ctx()).Tracef("nfqueue: marking packet %s (%s -> %s) on queue %d with %s after %s", pkt.ID(), pkt.Info().Src, pkt.Info().Dst, pkt.queue.id, markToString(mark), time.Since(pkt.Info().SeenAt)) + + // DEBUG: + // log.Tracer(pkt.Ctx()).Tracef( + // "nfqueue: marking packet %s (%s -> %s) on queue %d with %s after %s", + // pkt.ID(), pkt.Info().Src, pkt.Info().Dst, pkt.queue.id, + // markToString(mark), time.Since(pkt.Info().SeenAt), + // ) return nil } diff --git a/netquery/manager.go b/netquery/manager.go index e34baa9c..b6be97be 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -113,7 +113,8 @@ func (mng *Manager) HandleFeed(ctx context.Context, feed <-chan *network.Connect continue } - log.Tracef("netquery: updating connection %s", conn.ID) + // DEBUG: + // log.Tracef("netquery: updating connection %s", conn.ID) if err := mng.store.Save(ctx, *model, conn.HistoryEnabled); err != nil { log.Errorf("netquery: failed to save connection %s in sqlite database: %s", conn.ID, err) diff --git a/network/clean.go b/network/clean.go index a538b7f5..f3103142 100644 --- a/network/clean.go +++ b/network/clean.go @@ -78,7 +78,8 @@ func cleanConnections() (activePIDs map[int]struct{}) { } case conn.Ended < deleteOlderThan: // Step 3: delete - log.Tracef("network.clean: deleted %s (ended at %s)", conn.DatabaseKey(), time.Unix(conn.Ended, 0)) + // DEBUG: + // log.Tracef("network.clean: deleted %s (ended at %s)", conn.DatabaseKey(), time.Unix(conn.Ended, 0)) conn.delete() } From f0ebc6e72f2d5b6dc151b01a5c6df7d9c83ce16f Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:05:13 +0200 Subject: [PATCH 09/15] Update BytesReceived/Sent field names --- firewall/interception/ebpf/bandwidth/interface.go | 8 ++++---- .../interception/windowskext/bandwidth_stats.go | 8 ++++---- firewall/packet_handler.go | 14 +++++++------- netquery/database.go | 8 ++++---- network/connection.go | 5 +++++ network/packet/bandwidth.go | 10 +++++----- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/firewall/interception/ebpf/bandwidth/interface.go b/firewall/interception/ebpf/bandwidth/interface.go index f23d4452..53d2406d 100644 --- a/firewall/interception/ebpf/bandwidth/interface.go +++ b/firewall/interception/ebpf/bandwidth/interface.go @@ -133,10 +133,10 @@ func reportBandwidth(ctx context.Context, objs bpfObjects, bandwidthUpdates chan false, ) update := &packet.BandwidthUpdate{ - ConnID: connID, - RecvBytes: skInfo.Rx, - SentBytes: skInfo.Tx, - Method: packet.Absolute, + ConnID: connID, + BytesReceived: skInfo.Rx, + BytesSent: skInfo.Tx, + Method: packet.Absolute, } select { case bandwidthUpdates <- update: diff --git a/firewall/interception/windowskext/bandwidth_stats.go b/firewall/interception/windowskext/bandwidth_stats.go index 6e9dd05f..7147db97 100644 --- a/firewall/interception/windowskext/bandwidth_stats.go +++ b/firewall/interception/windowskext/bandwidth_stats.go @@ -63,10 +63,10 @@ func reportBandwidth(ctx context.Context, bandwidthUpdates chan *packet.Bandwidt false, ) update := &packet.BandwidthUpdate{ - ConnID: connID, - RecvBytes: stat.receivedBytes, - SentBytes: stat.transmittedBytes, - Method: packet.Additive, + ConnID: connID, + BytesReceived: stat.receivedBytes, + BytesSent: stat.transmittedBytes, + Method: packet.Additive, } select { case bandwidthUpdates <- update: diff --git a/firewall/packet_handler.go b/firewall/packet_handler.go index 133e7dba..0e70bcb9 100644 --- a/firewall/packet_handler.go +++ b/firewall/packet_handler.go @@ -629,7 +629,7 @@ func bandwidthUpdateHandler(ctx context.Context) error { func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { // Check if update makes sense. - if bwUpdate.RecvBytes == 0 && bwUpdate.SentBytes == 0 { + if bwUpdate.BytesReceived == 0 && bwUpdate.BytesSent == 0 { return } @@ -649,11 +649,11 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { // Update stats according to method. switch bwUpdate.Method { case packet.Absolute: - conn.RecvBytes = bwUpdate.RecvBytes - conn.SentBytes = bwUpdate.SentBytes + conn.BytesReceived = bwUpdate.BytesReceived + conn.BytesSent = bwUpdate.BytesSent case packet.Additive: - conn.RecvBytes += bwUpdate.RecvBytes - conn.SentBytes += bwUpdate.SentBytes + conn.BytesReceived += bwUpdate.BytesReceived + conn.BytesSent += bwUpdate.BytesSent default: log.Warningf("filter: unsupported bandwidth update method: %d", bwUpdate.Method) } @@ -664,8 +664,8 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { conn.HistoryEnabled, conn.Process().GetID(), conn.ID, - &conn.RecvBytes, - &conn.SentBytes, + conn.BytesReceived, + conn.BytesSent, ); err != nil { log.Errorf("firewall: failed to persist bandwidth data: %s", err) } diff --git a/netquery/database.go b/netquery/database.go index a5ce6c01..27207680 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -399,14 +399,14 @@ func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, pro } parts := []string{} - if incoming != nil { + if bytesReceived != 0 { parts = append(parts, "bytes_received = :bytes_received") - params[":bytes_received"] = *incoming + params[":bytes_received"] = bytesReceived } - if outgoing != nil { + if bytesSent != 0 { parts = append(parts, "bytes_sent = :bytes_sent") - params[":bytes_sent"] = *outgoing + params[":bytes_sent"] = bytesSent } updateSet := strings.Join(parts, ", ") diff --git a/network/connection.go b/network/connection.go index 3db4949d..dc32e43d 100644 --- a/network/connection.go +++ b/network/connection.go @@ -178,6 +178,11 @@ type Connection struct { //nolint:maligned // TODO: fix alignment RecvBytes uint64 SentBytes uint64 + // BytesReceived holds the observed received bytes of the connection. + BytesReceived uint64 + // BytesSent holds the observed sent bytes of the connection. + BytesSent uint64 + // pkgQueue is used to serialize packet handling for a single // connection and is served by the connections packetHandler. pktQueue chan packet.Packet diff --git a/network/packet/bandwidth.go b/network/packet/bandwidth.go index c65ac085..c2ce6a01 100644 --- a/network/packet/bandwidth.go +++ b/network/packet/bandwidth.go @@ -4,10 +4,10 @@ import "fmt" // BandwidthUpdate holds an update to the seen bandwidth of a connection. type BandwidthUpdate struct { - ConnID string - RecvBytes uint64 - SentBytes uint64 - Method BandwidthUpdateMethod + ConnID string + BytesReceived uint64 + BytesSent uint64 + Method BandwidthUpdateMethod } // BandwidthUpdateMethod defines how the bandwidth data of a bandwidth update should be interpreted. @@ -20,7 +20,7 @@ const ( ) func (bu *BandwidthUpdate) String() string { - return fmt.Sprintf("%s: %dB recv | %dB sent [%s]", bu.ConnID, bu.RecvBytes, bu.SentBytes, bu.Method) + return fmt.Sprintf("%s: %dB recv | %dB sent [%s]", bu.ConnID, bu.BytesReceived, bu.BytesSent, bu.Method) } func (bum BandwidthUpdateMethod) String() string { From 07f4253e0b52bcf38cb4a7f2b300ee875817e36d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:05:57 +0200 Subject: [PATCH 10/15] Improve logging and make linter happy --- .../interception/ebpf/bandwidth/interface.go | 3 +++ .../ebpf/connection_listener/worker.go | 2 ++ .../interception/windowskext/bandwidth_stats.go | 5 ++++- firewall/packet_handler.go | 8 +++++--- firewall/prompt.go | 4 ++-- netquery/database.go | 17 ++++++++++------- netquery/manager.go | 12 ++++++------ netquery/module_api.go | 6 +++--- netquery/orm/query_runner.go | 2 +- netquery/orm/schema_builder.go | 13 +++++++------ netquery/query.go | 2 ++ netquery/query_handler.go | 3 ++- 12 files changed, 47 insertions(+), 30 deletions(-) diff --git a/firewall/interception/ebpf/bandwidth/interface.go b/firewall/interception/ebpf/bandwidth/interface.go index 53d2406d..f247b157 100644 --- a/firewall/interception/ebpf/bandwidth/interface.go +++ b/firewall/interception/ebpf/bandwidth/interface.go @@ -142,6 +142,9 @@ func reportBandwidth(ctx context.Context, objs bpfObjects, bandwidthUpdates chan case bandwidthUpdates <- update: case <-ctx.Done(): return + default: + log.Warning("ebpf: bandwidth update queue is full, skipping rest of batch") + return } } } diff --git a/firewall/interception/ebpf/connection_listener/worker.go b/firewall/interception/ebpf/connection_listener/worker.go index 4798e389..1dee07be 100644 --- a/firewall/interception/ebpf/connection_listener/worker.go +++ b/firewall/interception/ebpf/connection_listener/worker.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "sync/atomic" + "time" "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/ringbuf" @@ -112,6 +113,7 @@ func ConnectionListenerWorker(ctx context.Context, packets chan packet.Packet) e Src: convertArrayToIPv4(event.Saddr, packet.IPVersion(event.IpVersion)), Dst: convertArrayToIPv4(event.Daddr, packet.IPVersion(event.IpVersion)), PID: int(event.Pid), + SeenAt: time.Now(), }) if isEventValid(event) { // DEBUG: diff --git a/firewall/interception/windowskext/bandwidth_stats.go b/firewall/interception/windowskext/bandwidth_stats.go index 7147db97..2a1bddc0 100644 --- a/firewall/interception/windowskext/bandwidth_stats.go +++ b/firewall/interception/windowskext/bandwidth_stats.go @@ -55,7 +55,7 @@ func reportBandwidth(ctx context.Context, bandwidthUpdates chan *packet.Bandwidt } // Report all statistics. - for _, stat := range stats { + for i, stat := range stats { connID := packet.CreateConnectionID( packet.IPProtocol(stat.protocol), convertArrayToIP(stat.localIP, stat.ipV6 == 1), stat.localPort, @@ -72,6 +72,9 @@ func reportBandwidth(ctx context.Context, bandwidthUpdates chan *packet.Bandwidt case bandwidthUpdates <- update: case <-ctx.Done(): return nil + default: + log.Warningf("kext: bandwidth update queue is full, skipping rest of batch (%d entries)", len(stats)-i) + return nil } } diff --git a/firewall/packet_handler.go b/firewall/packet_handler.go index 0e70bcb9..97df6eff 100644 --- a/firewall/packet_handler.go +++ b/firewall/packet_handler.go @@ -511,7 +511,7 @@ func issueVerdict(conn *network.Connection, pkt packet.Packet, verdict network.V atomic.AddUint64(packetsFailed, 1) err = pkt.Drop() case network.VerdictUndecided, network.VerdictUndeterminable: - log.Warningf("filter: tried to apply verdict %s to pkt %s: dropping instead", verdict, pkt) + log.Tracer(pkt.Ctx()).Warningf("filter: tried to apply verdict %s to pkt %s: dropping instead", verdict, pkt) fallthrough default: atomic.AddUint64(packetsDropped, 1) @@ -519,7 +519,7 @@ func issueVerdict(conn *network.Connection, pkt packet.Packet, verdict network.V } if err != nil { - log.Warningf("filter: failed to apply verdict to pkt %s: %s", pkt, err) + log.Tracer(pkt.Ctx()).Warningf("filter: failed to apply verdict to pkt %s: %s", pkt, err) } } @@ -656,8 +656,10 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { conn.BytesSent += bwUpdate.BytesSent default: log.Warningf("filter: unsupported bandwidth update method: %d", bwUpdate.Method) + return } + // Update bandwidth in the netquery module. if netquery.DefaultModule != nil && conn.BandwidthEnabled { if err := netquery.DefaultModule.Store.UpdateBandwidth( ctx, @@ -667,7 +669,7 @@ func updateBandwidth(ctx context.Context, bwUpdate *packet.BandwidthUpdate) { conn.BytesReceived, conn.BytesSent, ); err != nil { - log.Errorf("firewall: failed to persist bandwidth data: %s", err) + log.Errorf("filter: failed to persist bandwidth data: %s", err) } } } diff --git a/firewall/prompt.go b/firewall/prompt.go index e1a380d9..e3582ba0 100644 --- a/firewall/prompt.go +++ b/firewall/prompt.go @@ -91,12 +91,12 @@ func createPrompt(ctx context.Context, conn *network.Connection) (n *notificatio layeredProfile := conn.Process().Profile() if layeredProfile == nil { log.Tracer(ctx).Warningf("filter: tried creating prompt for connection without profile") - return + return nil } localProfile := layeredProfile.LocalProfile() if localProfile == nil { log.Tracer(ctx).Warningf("filter: tried creating prompt for connection without local profile") - return + return nil } // first check if there is an existing notification for this. diff --git a/netquery/database.go b/netquery/database.go index 27207680..f1abc633 100644 --- a/netquery/database.go +++ b/netquery/database.go @@ -112,7 +112,7 @@ type ( } ) -// New opens a new in-memory database named path. +// New opens a new in-memory database named path and attaches a persistent history database. // // The returned Database used connection pooling for read-only connections // (see Execute). To perform database writes use either Save() or ExecuteWrite(). @@ -131,7 +131,6 @@ func New(dbPath string) (*Database, error) { dbPath, sqlite.OpenReadOnly, sqlite.OpenSharedCache, - //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { @@ -171,7 +170,6 @@ func New(dbPath string) (*Database, error) { sqlite.OpenReadWrite, sqlite.OpenWAL, sqlite.OpenSharedCache, - //sqlite.OpenMemory, sqlite.OpenURI, ) if err != nil { @@ -337,11 +335,14 @@ func (db *Database) Cleanup(ctx context.Context, threshold time.Time) (int, erro return result[0].Count, nil } +// RemoveAllHistoryData removes all connections from the history database. func (db *Database) RemoveAllHistoryData(ctx context.Context) error { query := fmt.Sprintf("DELETE FROM %s.connections", HistoryDatabase) return db.ExecuteWrite(ctx, query) } +// RemoveHistoryForProfile removes all connections from the history database +// for a given profile ID (source/id). func (db *Database) RemoveHistoryForProfile(ctx context.Context, profileID string) error { query := fmt.Sprintf("DELETE FROM %s.connections WHERE profile = :profile", HistoryDatabase) return db.ExecuteWrite(ctx, query, orm.WithNamedArgs(map[string]any{ @@ -389,13 +390,15 @@ func (db *Database) MarkAllHistoryConnectionsEnded(ctx context.Context) error { return nil } -func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, incoming *uint64, outgoing *uint64) error { +// UpdateBandwidth updates bandwidth data for the connection and optionally also writes +// the bandwidth data to the history database. +func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error { data := connID + "-" + processKey hash := sha256.Sum256([]byte(data)) - dbConnId := hex.EncodeToString(hash[:]) + dbConnID := hex.EncodeToString(hash[:]) params := map[string]any{ - ":id": dbConnId, + ":id": dbConnID, } parts := []string{} @@ -439,7 +442,7 @@ func (db *Database) UpdateBandwidth(ctx context.Context, enableHistory bool, pro func (db *Database) Save(ctx context.Context, conn Conn, enableHistory bool) error { // convert the connection to a param map where each key is already translated // to the sql column name. We also skip bytes_received and bytes_sent since those - // will be updated independenly from the connection object. + // will be updated independently from the connection object. connMap, err := orm.ToParamMap(ctx, conn, "", orm.DefaultEncodeConfig, []string{ "bytes_received", "bytes_sent", diff --git a/netquery/manager.go b/netquery/manager.go index b6be97be..c49aa5c2 100644 --- a/netquery/manager.go +++ b/netquery/manager.go @@ -31,16 +31,16 @@ type ( // database as ended NOW. MarkAllHistoryConnectionsEnded(context.Context) error - // RemoveHistoryForProfile removes all connections from the history database - // for a given profile ID (source/id) - RemoveHistoryForProfile(context.Context, string) error - // RemoveAllHistoryData removes all connections from the history database. RemoveAllHistoryData(context.Context) error - // UpdateBandwidth updates bandwith data for the connection and optionally also writes + // RemoveHistoryForProfile removes all connections from the history database. + // for a given profile ID (source/id) + RemoveHistoryForProfile(context.Context, string) error + + // UpdateBandwidth updates bandwidth data for the connection and optionally also writes // the bandwidth data to the history database. - UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, incoming *uint64, outgoing *uint64) error + UpdateBandwidth(ctx context.Context, enableHistory bool, processKey string, connID string, bytesReceived uint64, bytesSent uint64) error } // Manager handles new and updated network.Connections feeds and persists them diff --git a/netquery/module_api.go b/netquery/module_api.go index f102ec65..344f9391 100644 --- a/netquery/module_api.go +++ b/netquery/module_api.go @@ -8,6 +8,7 @@ import ( "time" "github.com/hashicorp/go-multierror" + "github.com/safing/portbase/api" "github.com/safing/portbase/config" "github.com/safing/portbase/database" @@ -19,6 +20,7 @@ import ( "github.com/safing/portmaster/network" ) +// DefaultModule is the default netquery module. var DefaultModule *module type module struct { @@ -120,8 +122,6 @@ func (m *module) prepare() error { ProfileIDs []string `json:"profileIDs"` } - defer r.Body.Close() - dec := json.NewDecoder(r.Body) dec.DisallowUnknownFields() @@ -240,7 +240,7 @@ func (m *module) stop() error { if err := m.mng.store.MarkAllHistoryConnectionsEnded(ctx); err != nil { // handle the error by just logging it. There's not much we can do here // and returning an error to the module system doesn't help much as well... - log.Errorf("failed to mark connections in history database as eded: %w", err) + log.Errorf("netquery: failed to mark connections in history database as ended: %s", err) } return nil diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go index f59cca79..135a29f6 100644 --- a/netquery/orm/query_runner.go +++ b/netquery/orm/query_runner.go @@ -148,7 +148,7 @@ func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...Q for colIdx := 0; colIdx < stmt.ColumnCount(); colIdx++ { name := stmt.ColumnName(colIdx) - switch stmt.ColumnType(colIdx) { + switch stmt.ColumnType(colIdx) { //nolint:exhaustive // TODO: handle type BLOB? case sqlite.TypeText: resultDump[name] = stmt.ColumnText(colIdx) case sqlite.TypeFloat: diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go index 080c5003..6aba2a1f 100644 --- a/netquery/orm/schema_builder.go +++ b/netquery/orm/schema_builder.go @@ -7,8 +7,9 @@ import ( "strconv" "strings" - "github.com/safing/portbase/log" "zombiezen.com/go/sqlite" + + "github.com/safing/portbase/log" ) var errSkipStructField = errors.New("struct field should be skipped") @@ -110,13 +111,13 @@ func (def ColumnDef) AsSQL() string { } if def.Default != nil { sql += " DEFAULT " - switch def.Type { + switch def.Type { //nolint:exhaustive // TODO: handle types BLOB, NULL? case sqlite.TypeFloat: - sql += strconv.FormatFloat(def.Default.(float64), 'b', 0, 64) + sql += strconv.FormatFloat(def.Default.(float64), 'b', 0, 64) //nolint:forcetypeassert case sqlite.TypeInteger: - sql += strconv.FormatInt(def.Default.(int64), 10) + sql += strconv.FormatInt(def.Default.(int64), 10) //nolint:forcetypeassert case sqlite.TypeText: - sql += fmt.Sprintf("%q", def.Default.(string)) + sql += fmt.Sprintf("%q", def.Default.(string)) //nolint:forcetypeassert default: log.Errorf("unsupported default value: %q %q", def.Type, def.Default) sql = strings.TrimSuffix(sql, " DEFAULT ") @@ -257,7 +258,7 @@ func applyStructFieldTag(fieldType reflect.StructField, def *ColumnDef) error { if strings.HasPrefix(k, TagTypePrefixDefault) { defaultValue := strings.TrimPrefix(k, TagTypePrefixDefault) - switch def.Type { + switch def.Type { //nolint:exhaustive case sqlite.TypeFloat: fv, err := strconv.ParseFloat(defaultValue, 64) if err != nil { diff --git a/netquery/query.go b/netquery/query.go index 264f0bd7..06b766f6 100644 --- a/netquery/query.go +++ b/netquery/query.go @@ -14,8 +14,10 @@ import ( "github.com/safing/portmaster/netquery/orm" ) +// DatabaseName is a database name constant. type DatabaseName string +// Databases. const ( LiveDatabase = DatabaseName("main") HistoryDatabase = DatabaseName("history") diff --git a/netquery/query_handler.go b/netquery/query_handler.go index 3c6bb453..e555965d 100644 --- a/netquery/query_handler.go +++ b/netquery/query_handler.go @@ -12,9 +12,10 @@ import ( "strings" "time" + "golang.org/x/exp/slices" + "github.com/safing/portbase/log" "github.com/safing/portmaster/netquery/orm" - "golang.org/x/exp/slices" ) var charOnlyRegexp = regexp.MustCompile("[a-zA-Z]+") From daa33c1a8898264ab2efc23ec3403f4f7ba221bf Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:06:21 +0200 Subject: [PATCH 11/15] Improve network history setting --- firewall/config.go | 4 ++-- profile/config.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/firewall/config.go b/firewall/config.go index eaf3fa44..ea1785b3 100644 --- a/firewall/config.go +++ b/firewall/config.go @@ -23,11 +23,11 @@ var ( askTimeout config.IntOption CfgOptionPermanentVerdictsKey = "filter/permanentVerdicts" - cfgOptionPermanentVerdictsOrder = 96 + cfgOptionPermanentVerdictsOrder = 80 permanentVerdicts config.BoolOption CfgOptionDNSQueryInterceptionKey = "filter/dnsQueryInterception" - cfgOptionDNSQueryInterceptionOrder = 97 + cfgOptionDNSQueryInterceptionOrder = 81 dnsQueryInterception config.BoolOption ) diff --git a/profile/config.go b/profile/config.go index 424b459a..ff3e072c 100644 --- a/profile/config.go +++ b/profile/config.go @@ -104,11 +104,13 @@ var ( cfgOptionDisableAutoPermit config.IntOption // security level option cfgOptionDisableAutoPermitOrder = 65 - // Setting "Permanent Verdicts" at order 96. + // Setting "Permanent Verdicts" at order 80. - CfgOptionEnableHistoryKey = "history/enabled" + // Network History. + + CfgOptionEnableHistoryKey = "history/enable" cfgOptionEnableHistory config.BoolOption - cfgOptionEnableHistoryOrder = 66 + cfgOptionEnableHistoryOrder = 96 // Setting "Enable SPN" at order 128. From a5a5a15112a50d2d1f2e722552731860f39fdd2e Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 21 Jul 2023 16:06:47 +0200 Subject: [PATCH 12/15] Improve updating allowed features on connection --- network/connection.go | 69 ++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/network/connection.go b/network/connection.go index dc32e43d..972a5c5b 100644 --- a/network/connection.go +++ b/network/connection.go @@ -175,8 +175,12 @@ type Connection struct { //nolint:maligned // TODO: fix alignment StopTunnel() error } - RecvBytes uint64 - SentBytes uint64 + // HistoryEnabled is set to true when the connection should be persisted + // in the history database. + HistoryEnabled bool + // BanwidthEnabled is set to true if connection bandwidth data should be persisted + // in netquery. + BandwidthEnabled bool // BytesReceived holds the observed received bytes of the connection. BytesReceived uint64 @@ -225,13 +229,6 @@ type Connection struct { //nolint:maligned // TODO: fix alignment // addedToMetrics signifies if the connection has already been counted in // the metrics. addedToMetrics bool - - // HistoryEnabled is set to true when the connection should be persisted - // in the history database. - HistoryEnabled bool - // BanwidthEnabled is set to true if connection bandwidth data should be persisted - // in netquery. - BandwidthEnabled bool } // Reason holds information justifying a verdict, as well as additional @@ -340,6 +337,10 @@ func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, cnames []stri // Inherit internal status of profile. if localProfile := proc.Profile().LocalProfile(); localProfile != nil { dnsConn.Internal = localProfile.Internal + + if err := dnsConn.updateFeatures(); err != nil { + log.Tracer(ctx).Warningf("network: failed to check for enabled features: %s", err) + } } // DNS Requests are saved by the nameserver depending on the result of the @@ -378,6 +379,10 @@ func NewConnectionFromExternalDNSRequest(ctx context.Context, fqdn string, cname // Inherit internal status of profile. if localProfile := remoteHost.Profile().LocalProfile(); localProfile != nil { dnsConn.Internal = localProfile.Internal + + if err := dnsConn.updateFeatures(); err != nil { + log.Tracer(ctx).Warningf("network: failed to check for enabled features: %s", err) + } } // DNS Requests are saved by the nameserver depending on the result of the @@ -388,6 +393,8 @@ func NewConnectionFromExternalDNSRequest(ctx context.Context, fqdn string, cname return dnsConn, nil } +var tooOldTimestamp = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix() + // NewIncompleteConnection creates a new incomplete connection with only minimal information. func NewIncompleteConnection(pkt packet.Packet) *Connection { info := pkt.Info() @@ -404,6 +411,12 @@ func NewIncompleteConnection(pkt packet.Packet) *Connection { dataComplete: abool.NewBool(false), } + // Bullshit check Started timestamp. + if conn.Started < tooOldTimestamp { + // Fix timestamp, use current time as fallback. + conn.Started = time.Now().Unix() + } + // Save connection to internal state in order to mitigate creation of // duplicates. Do not propagate yet, as data is not yet complete. conn.UpdateMeta() @@ -435,17 +448,8 @@ func (conn *Connection) GatherConnectionInfo(pkt packet.Packet) (err error) { if localProfile := conn.process.Profile().LocalProfile(); localProfile != nil { conn.Internal = localProfile.Internal - // check if we should persist the connection in the history database. - // Also make sure the current SPN User/subscription allows use of the history. - user, err := access.GetUser() - if err == nil { - if user.MayUse(account.FeatureHistory) { - conn.HistoryEnabled = localProfile.HistoryEnabled() - } - - if user.MayUse(account.FeatureBWVis) { - conn.BandwidthEnabled = true - } + if err := conn.updateFeatures(); err != nil { + log.Tracer(pkt.Ctx()).Warningf("network: failed to check for enabled features: %s", err) } } @@ -561,6 +565,31 @@ func (conn *Connection) SetLocalIP(ip net.IP) { conn.LocalIPScope = netutils.GetIPScope(ip) } +// updateFeatures checks which connection related features may be used and sets +// the flags accordingly. +func (conn *Connection) updateFeatures() error { + // Get user. + user, err := access.GetUser() + if err != nil { + return err + } + + // Check if history may be used and if it is enabled for this application. + if user.MayUse(account.FeatureHistory) { + lProfile := conn.Process().Profile() + if lProfile != nil { + conn.HistoryEnabled = lProfile.HistoryEnabled() + } + } + + // Check if bandwidth visibility may be used. + if user.MayUse(account.FeatureBWVis) { + conn.BandwidthEnabled = true + } + + return nil +} + // AcceptWithContext accepts the connection. func (conn *Connection) AcceptWithContext(reason, reasonOptionKey string, ctx interface{}) { if !conn.SetVerdict(VerdictAccept, reason, reasonOptionKey, ctx) { From 3cc12a3d69e72bd1bd22bf52860bcb1a26073a9f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 22 Jul 2023 20:23:33 +0200 Subject: [PATCH 13/15] Increase timeout of self-check --- compat/selfcheck.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/selfcheck.go b/compat/selfcheck.go index c1508d12..4515d93c 100644 --- a/compat/selfcheck.go +++ b/compat/selfcheck.go @@ -28,12 +28,12 @@ var ( systemIntegrationCheckDialNet = fmt.Sprintf("ip4:%d", uint8(SystemIntegrationCheckProtocol)) systemIntegrationCheckDialIP = SystemIntegrationCheckDstIP.String() systemIntegrationCheckPackets = make(chan packet.Packet, 1) - systemIntegrationCheckWaitDuration = 20 * time.Second + systemIntegrationCheckWaitDuration = 40 * time.Second // DNSCheckInternalDomainScope is the domain scope to use for dns checks. DNSCheckInternalDomainScope = ".self-check." + resolver.InternalSpecialUseDomain dnsCheckReceivedDomain = make(chan string, 1) - dnsCheckWaitDuration = 20 * time.Second + dnsCheckWaitDuration = 40 * time.Second dnsCheckAnswerLock sync.Mutex dnsCheckAnswer net.IP ) From c6569e64b101cb00d7f929e0e3a045e6b05fa93d Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 22 Jul 2023 20:23:59 +0200 Subject: [PATCH 14/15] Update SPN to v0.6.9 --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 69483277..a046cc5d 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/safing/jess v0.3.1 github.com/safing/portbase v0.17.0 github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 - github.com/safing/spn v0.6.8 + github.com/safing/spn v0.6.9 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.7.0 github.com/spkg/zipfs v0.7.1 @@ -28,6 +28,7 @@ require ( github.com/tannerryan/ring v1.1.2 github.com/tevino/abool v1.2.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/net v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.10.0 @@ -86,7 +87,6 @@ require ( github.com/zalando/go-keyring v0.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.11.0 // indirect diff --git a/go.sum b/go.sum index 61148a1b..ef2bba87 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/safing/portbase v0.17.0 h1:RsDzbCGxRIbgaArri3y7MZskfxytEvvkzJpiboDUER github.com/safing/portbase v0.17.0/go.mod h1:eKCRqsfMFLVhNpd2sY/fKvnbuk+LrIYnQEZCg1i86Ho= github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 h1:olc/REnUdpJN/Gmz8B030OxLpMYxyPDTrDILNEw0eKs= github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE= -github.com/safing/spn v0.6.8 h1:2obvyMzyw5X3CIYedLBE88kNBBrJumF84q1qtQSFqkc= -github.com/safing/spn v0.6.8/go.mod h1:Mh9bmkqFhO/dHNi9RWXzoXjQij893I4Lj8Wn4tQ0KZA= +github.com/safing/spn v0.6.9 h1:CCRN5jgshJrLBHwGHl0ywWwhukc+Wff7/I66qgYyymg= +github.com/safing/spn v0.6.9/go.mod h1:Mh9bmkqFhO/dHNi9RWXzoXjQij893I4Lj8Wn4tQ0KZA= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0= From e18d7ade3dcc064b9a3f32da0a5d51b840d2cfd8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 22 Jul 2023 20:42:26 +0200 Subject: [PATCH 15/15] Add missing method comment --- process/process.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/process/process.go b/process/process.go index 99c281cc..9f0acc7e 100644 --- a/process/process.go +++ b/process/process.go @@ -313,6 +313,9 @@ func loadProcess(ctx context.Context, key string, pInfo *processInfo.Process) (* return process, nil } +// GetID returns the key that is used internally to identify the process. +// The ID consists of the PID and the start time of the process as reported by +// the system. func (p *Process) GetID() string { return p.processKey }