github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite network tree saving and cleaning procedures

Browse Source
This commit is contained in:
Daniel
2019-05-22 16:10:05 +02:00
parent 1873999b38
commit fb4fb20d4b
9 changed files with 312 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
network/clean.go
View File

@@ -5,35 +5,33 @@ package network
import (
"time"
"github.com/Safing/portbase/log"
"github.com/Safing/portmaster/process"
)
var (
cleanerTickDuration = 10 * time.Second
deadLinksTimeout = 3 * time.Minute
thresholdDuration = 3 * time.Minute
cleanerTickDuration = 10 * time.Second
deleteLinksAfterEndedThreshold = 5 * time.Minute
deleteCommsWithoutLinksThreshhold = 3 * time.Minute
lastEstablishedUpdateThreshold = 30 * time.Second
)
func cleaner() {
for {
time.Sleep(cleanerTickDuration)
cleanLinks()
time.Sleep(2 * time.Second)
cleanComms()
time.Sleep(2 * time.Second)
cleanProcesses()
activeComms := cleanLinks()
activeProcs := cleanComms(activeComms)
process.CleanProcessStorage(activeProcs)
}
}
func cleanLinks() {
func cleanLinks() (activeComms map[string]struct{}) {
activeComms = make(map[string]struct{})
activeIDs := process.GetActiveConnectionIDs()
now := time.Now().Unix()
deleteOlderThan := time.Now().Add(-deadLinksTimeout).Unix()
// log.Tracef("network.clean: now=%d", now)
// log.Tracef("network.clean: deleteOlderThan=%d", deleteOlderThan)
deleteOlderThan := time.Now().Add(-deleteLinksAfterEndedThreshold).Unix()
linksLock.RLock()
defer linksLock.RUnlock()
@@ -42,18 +40,21 @@ func cleanLinks() {
for key, link := range links {
// delete dead links
if link.Ended > 0 {
link.Lock()
deleteThis := link.Ended < deleteOlderThan
link.Unlock()
if deleteThis {
// log.Tracef("network.clean: deleted %s", link.DatabaseKey())
go link.Delete()
}
link.Lock()
deleteThis := link.Ended > 0 && link.Ended < deleteOlderThan
link.Unlock()
if deleteThis {
log.Tracef("network.clean: deleted %s (ended at %d)", link.DatabaseKey(), link.Ended)
go link.Delete()
continue
}
// not yet deleted, so its still a valid link regarding link count
comm := link.Communication()
comm.Lock()
markActive(activeComms, comm.DatabaseKey())
comm.Unlock()
// check if link is dead
found = false
for _, activeID := range activeIDs {
@@ -63,31 +64,53 @@ func cleanLinks() {
}
}
// mark end time
if !found {
// mark end time
link.Lock()
link.Ended = now
// log.Tracef("network.clean: marked %s as ended.", link.DatabaseKey())
go link.Save()
link.Unlock()
log.Tracef("network.clean: marked %s as ended", link.DatabaseKey())
go link.save()
}
}
return
}
func cleanComms() {
func cleanComms(activeLinks map[string]struct{}) (activeComms map[string]struct{}) {
activeComms = make(map[string]struct{})
commsLock.RLock()
defer commsLock.RUnlock()
threshold := time.Now().Add(-thresholdDuration).Unix()
threshold := time.Now().Add(-deleteCommsWithoutLinksThreshhold).Unix()
for _, comm := range comms {
// has links?
_, hasLinks := activeLinks[comm.DatabaseKey()]
// comm created
comm.Lock()
if comm.FirstLinkEstablished < threshold && comm.LinkCount == 0 {
// log.Tracef("network.clean: deleted %s", comm.DatabaseKey())
go comm.Delete()
}
created := comm.Meta().Created
comm.Unlock()
if !hasLinks && created < threshold {
log.Tracef("network.clean: deleted %s", comm.DatabaseKey())
go comm.Delete()
} else {
p := comm.Process()
p.Lock()
markActive(activeComms, p.DatabaseKey())
p.Unlock()
}
}
return
}
func cleanProcesses() {
process.CleanProcessStorage(thresholdDuration)
func markActive(activeMap map[string]struct{}, key string) {
_, ok := activeMap[key]
if !ok {
activeMap[key] = struct{}{}
}
}

89
network/communication.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"github.com/Safing/portbase/database/record"
"github.com/Safing/portbase/log"
"github.com/Safing/portmaster/intel"
"github.com/Safing/portmaster/network/netutils"
"github.com/Safing/portmaster/network/packet"
@@ -33,9 +34,9 @@ type Communication struct {
FirstLinkEstablished int64
LastLinkEstablished int64
LinkCount uint
profileUpdateVersion uint32
saveWhenFinished bool
}
// Process returns the process that owns the connection.
@@ -53,6 +54,7 @@ func (comm *Communication) ResetVerdict() {
comm.Verdict = VerdictUndecided
comm.Reason = ""
comm.saveWhenFinished = true
}
// GetVerdict returns the current verdict.
@@ -97,7 +99,7 @@ func (comm *Communication) UpdateVerdict(newVerdict Verdict) {
if newVerdict > comm.Verdict {
comm.Verdict = newVerdict
go comm.Save()
comm.saveWhenFinished = true
}
}
@@ -110,6 +112,7 @@ func (comm *Communication) SetReason(reason string) {
comm.Lock()
defer comm.Unlock()
comm.Reason = reason
comm.saveWhenFinished = true
}
// AddReason adds a human readable string as to why a certain verdict was set in regard to this communication.
@@ -174,6 +177,7 @@ func GetCommunicationByFirstPacket(pkt packet.Packet) (*Communication, error) {
process: proc,
Inspect: true,
FirstLinkEstablished: time.Now().Unix(),
saveWhenFinished: true,
}
}
communication.process.AddCommunication()
@@ -206,6 +210,7 @@ func GetCommunicationByFirstPacket(pkt packet.Packet) (*Communication, error) {
process: proc,
Inspect: true,
FirstLinkEstablished: time.Now().Unix(),
saveWhenFinished: true,
}
}
communication.process.AddCommunication()
@@ -222,6 +227,7 @@ func GetCommunicationByFirstPacket(pkt packet.Packet) (*Communication, error) {
process: proc,
Inspect: true,
FirstLinkEstablished: time.Now().Unix(),
saveWhenFinished: true,
}
}
communication.process.AddCommunication()
@@ -246,12 +252,13 @@ func GetCommunicationByDNSRequest(ctx context.Context, ip net.IP, port uint16, f
communication, ok := GetCommunication(proc.Pid, fqdn)
if !ok {
communication = &Communication{
Domain: fqdn,
process: proc,
Inspect: true,
Domain: fqdn,
process: proc,
Inspect: true,
saveWhenFinished: true,
}
communication.process.AddCommunication()
communication.Save()
communication.saveWhenFinished = true
}
return communication, nil
}
@@ -268,21 +275,47 @@ func (comm *Communication) makeKey() string {
return fmt.Sprintf("%d/%s", comm.process.Pid, comm.Domain)
}
// Save saves the connection object in the storage and propagates the change.
func (comm *Communication) Save() error {
comm.Lock()
defer comm.Unlock()
// SaveWhenFinished marks the Connection for saving after all current actions are finished.
func (comm *Communication) SaveWhenFinished() {
comm.saveWhenFinished = true
}
// SaveIfNeeded saves the Connection if it is marked for saving when finished.
func (comm *Communication) SaveIfNeeded() {
comm.Lock()
save := comm.saveWhenFinished
if save {
comm.saveWhenFinished = false
}
comm.Unlock()
if save {
comm.save()
}
}
// Save saves the Connection object in the storage and propagates the change.
func (comm *Communication) save() error {
// update comm
comm.Lock()
if comm.process == nil {
comm.Unlock()
return errors.New("cannot save connection without process")
}
if !comm.KeyIsSet() {
comm.SetKey(fmt.Sprintf("network:tree/%d/%s", comm.process.Pid, comm.Domain))
comm.CreateMeta()
comm.UpdateMeta()
}
if comm.Meta().Deleted > 0 {
log.Criticalf("network: revieving dead comm %s", comm)
comm.Meta().Deleted = 0
}
key := comm.makeKey()
comm.saveWhenFinished = false
comm.Unlock()
// save comm
commsLock.RLock()
_, ok := comms[key]
commsLock.RUnlock()
@@ -299,46 +332,42 @@ func (comm *Communication) Save() error {
// Delete deletes a connection from the storage and propagates the change.
func (comm *Communication) Delete() {
commsLock.Lock()
defer commsLock.Unlock()
comm.Lock()
defer comm.Unlock()
commsLock.Lock()
delete(comms, comm.makeKey())
commsLock.Unlock()
comm.Meta().Delete()
go dbController.PushUpdate(comm)
comm.process.RemoveCommunication()
go comm.process.Save()
}
// AddLink applies the Communication to the Link and increases sets counter and timestamps.
// AddLink applies the Communication to the Link and sets timestamps.
func (comm *Communication) AddLink(link *Link) {
// apply comm to link
link.Lock()
link.comm = comm
link.Verdict = comm.Verdict
link.Inspect = comm.Inspect
link.saveWhenFinished = true
link.Unlock()
link.Save()
// update comm LastLinkEstablished
comm.Lock()
comm.LinkCount++
// check if we should save
if comm.LastLinkEstablished < time.Now().Add(-3*time.Second).Unix() {
comm.saveWhenFinished = true
}
// update LastLinkEstablished
comm.LastLinkEstablished = time.Now().Unix()
if comm.FirstLinkEstablished == 0 {
comm.FirstLinkEstablished = comm.LastLinkEstablished
}
comm.Unlock()
comm.Save()
}
// RemoveLink lowers the link counter by one.
func (comm *Communication) RemoveLink() {
comm.Lock()
defer comm.Unlock()
if comm.LinkCount > 0 {
comm.LinkCount--
}
}
// String returns a string representation of Communication.

47
network/database.go
View File

@@ -1,6 +1,7 @@
package network
import (
"fmt"
"strconv"
"strings"
"sync"
@@ -14,9 +15,9 @@ import (
)
var (
links = make(map[string]*Link)
links = make(map[string]*Link) // key: Link ID
linksLock sync.RWMutex
comms = make(map[string]*Communication)
comms = make(map[string]*Communication) // key: PID/Domain
commsLock sync.RWMutex
dbController *database.Controller
@@ -45,7 +46,7 @@ func (s *StorageInterface) Get(key string) (record.Record, error) {
case 3:
commsLock.RLock()
defer commsLock.RUnlock()
conn, ok := comms[splitted[2]]
conn, ok := comms[fmt.Sprintf("%s/%s", splitted[1], splitted[2])]
if ok {
return conn, nil
}
@@ -72,30 +73,38 @@ func (s *StorageInterface) Query(q *query.Query, local, internal bool) (*iterato
}
func (s *StorageInterface) processQuery(q *query.Query, it *iterator.Iterator) {
// processes
for _, proc := range process.All() {
if strings.HasPrefix(proc.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- proc
slashes := strings.Count(q.DatabaseKeyPrefix(), "/")
if slashes <= 1 {
// processes
for _, proc := range process.All() {
if strings.HasPrefix(proc.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- proc
}
}
}
// comms
commsLock.RLock()
for _, conn := range comms {
if strings.HasPrefix(conn.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- conn
if slashes <= 2 {
// comms
commsLock.RLock()
for _, conn := range comms {
if strings.HasPrefix(conn.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- conn
}
}
commsLock.RUnlock()
}
commsLock.RUnlock()
// links
linksLock.RLock()
for _, link := range links {
if strings.HasPrefix(link.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- link
if slashes <= 3 {
// links
linksLock.RLock()
for _, link := range links {
if strings.HasPrefix(link.DatabaseKey(), q.DatabaseKeyPrefix()) {
it.Next <- link
}
}
linksLock.RUnlock()
}
linksLock.RUnlock()
it.Finish(nil)
}

44
network/link.go
View File

@@ -42,6 +42,7 @@ type Link struct {
activeInspectors []bool
inspectorData map[uint8]interface{}
saveWhenFinished bool
}
// Communication returns the Communication the Link is part of
@@ -148,7 +149,7 @@ func (link *Link) UpdateVerdict(newVerdict Verdict) {
if newVerdict > link.Verdict {
link.Verdict = newVerdict
go link.Save()
link.saveWhenFinished = true
}
}
@@ -165,6 +166,8 @@ func (link *Link) AddReason(reason string) {
link.Reason += " | "
}
link.Reason += reason
link.saveWhenFinished = true
}
// packetHandler sequentially handles queued packets
@@ -223,20 +226,42 @@ func (link *Link) ApplyVerdict(pkt packet.Packet) {
}
}
// Save saves the link object in the storage and propagates the change.
func (link *Link) Save() error {
link.Lock()
defer link.Unlock()
// SaveWhenFinished marks the Link for saving after all current actions are finished.
func (link *Link) SaveWhenFinished() {
link.saveWhenFinished = true
}
// SaveIfNeeded saves the Link if it is marked for saving when finished.
func (link *Link) SaveIfNeeded() {
link.Lock()
save := link.saveWhenFinished
if save {
link.saveWhenFinished = false
}
link.Unlock()
if save {
link.save()
}
}
// Save saves the link object in the storage and propagates the change.
func (link *Link) save() error {
// update link
link.Lock()
if link.comm == nil {
link.Unlock()
return errors.New("cannot save link without comms")
}
if !link.KeyIsSet() {
link.SetKey(fmt.Sprintf("network:tree/%d/%s/%s", link.comm.Process().Pid, link.comm.Domain, link.ID))
link.CreateMeta()
link.UpdateMeta()
}
link.saveWhenFinished = false
link.Unlock()
// save link
linksLock.RLock()
_, ok := links[link.ID]
linksLock.RUnlock()
@@ -253,17 +278,15 @@ func (link *Link) Save() error {
// Delete deletes a link from the storage and propagates the change.
func (link *Link) Delete() {
linksLock.Lock()
defer linksLock.Unlock()
link.Lock()
defer link.Unlock()
linksLock.Lock()
delete(links, link.ID)
linksLock.Unlock()
link.Meta().Delete()
go dbController.PushUpdate(link)
link.comm.RemoveLink()
go link.comm.Save()
}
// GetLink fetches a Link from the database from the default namespace for this object
@@ -294,6 +317,7 @@ func CreateLinkFromPacket(pkt packet.Packet) *Link {
Verdict: VerdictUndecided,
Started: time.Now().Unix(),
RemoteAddress: pkt.FmtRemoteAddress(),
saveWhenFinished: true,
}
return link
}

2
network/unknown.go
View File

@@ -52,7 +52,7 @@ func getOrCreateUnknownCommunication(pkt packet.Packet, connClass string) (*Comm
Verdict: VerdictDrop,
Reason: ReasonUnknownProcess,
process: process.UnknownProcess,
Inspect: true,
Inspect: false,
FirstLinkEstablished: time.Now().Unix(),
}
if pkt.IsOutbound() {