diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..b6462b80 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,53 @@ +--- +name: Bug Report +about: Report a bug encountered while using the Portmaster +labels: bug + +--- + + + +**Checklist**: + +- [ ] I'm using the official portmaster release (i.e no custom builds) +- [ ] I searched for similar/existing issues first. +- [ ] My issue is not mentioned in the Known Issues section of my [OS](https://github.com/safing/portmaster/wiki) + +**What happened**: + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + +**Anything else we need to know?**: + +**Environment**: + +Portmaster Version: + +
+ Versions from the `About` page in Portmaster's UI + +
+ +Operating System: +- [ ] Windows 7 +- [ ] Windows 8/8.1 +- [ ] Windows 10 +- [ ] Linux + - Please provide the output of `cat /etc/os-release` + +If applicable you can provide related sections from the log files and ensure to **remove sensitive or otherwise private information**. + - Linux: `/var/lib/portmaster/logs` + - Windows: `%PROGRAMDATA%\Portmaster\ļogs` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..4ab1dc2b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,13 @@ + +# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser +blank_issues_enabled: true # default: true +contact_links: +- name: User Interface Issues 💻 + url: https://github.com/safing/portmaster-ui/issues/new/choose + about: Report any issues or feature requests for the Portmaster UI here +- name: Support Requests & Community 🤷 + url: https://www.reddit.com/r/safing + about: Ask for support and any questions you might have on reddit. +- name: Code of Conduct 📝 + url: https://github.com/safing/portmaster/blob/develop/CODE_OF_CONDUCT.md + about: Be nice to other community members ❤ diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 00000000..87f04413 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,15 @@ +--- +name: Feature Request +about: Suggest an enhancement or feature to the Portmaster +labels: feature + +--- + + +**What would you like to be added**: + +**Why is this needed**: diff --git a/README.md b/README.md index b8caab10..2ca6215a 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,54 @@ # Portmaster -The Portmaster enables you to protect your data on your device. You are back in charge of your outgoing connections: you choose what data you share and what data stays private. Read more on [docs.safing.io](http://docs.safing.io/). +The Portmaster is a privacy app that at its core simply intercepts _all_ your network connections. Different modules with different privacy features are built on top of it, which can all be enabled or disabled as desired: -## Current Status +![portmaster modules](https://safing.io/assets/img/portmaster/modules.png) -**NOTE: Portmaster is currently in development freeze in order to focus on our upcoming [privacy network](https://safing.io/technology/#gate17) (Codename: Gate17)** +#### ⚠️ Disclaimer -The Portmaster is currently in alpha. Expect dragons. -Supported platforms: +> The Portmaster is still in its early "pre-alpha" development stage. It is functional, but has not yet been tested widely. We are glad if you want to try out the Portmaster right away but please expect bugs and rather technical problems. We'll push updates and fixes as we go. A list of known problems can be found at the bottom of this page. -- linux_amd64 -- windows_amd64 (_soon_) -- darwin_amd64 (_later_) +#### 🔄 Automatic Updates -## Using the Alpha Version +We have set up update servers so we can push fixes and improvements as we go. -#### Must-Know Basics +# Modules -The Portmaster is all about protecting your privacy. As soon as it starts, it will start to intercept network connections. If other programs are already running, this may cause them to lose Internet connectivity for a short duration. +## DNS-over-TLS Resolver -The main way to configure the application firewall is by configuring application profiles. For every program that is active on the network the Portmaster automatically creates a profile for it the first it's seen. These profiles are empty at first and only fed by a fallback profile. By changing these profiles in the app, you change what programs are allowed to do. +**Status:** _pre-alpha_ -You can also see what is going on right now. The monitor page in the app lets you see the network like the Portmaster sees it: `Communications` represent a logical connection between a program and a domain. These second level objects group `Links` (physical connections: IP->IP) together for easier handling and viewing. +A DNS resolver that does not only encrypt your queries, but figures out where it makes the most sense to send your queries. Queries for local domains will not be sent to the upstream servers. This means it won't break your or your company's network setup. -The Portmaster consists of three parts: -- The _core_ (ie. the _daemon_) that runs as an administrator and does all the work. (`sudo ./pmctl run core --data=/opt/pm_db`) -- The _app_, a user interface to set preferences, monitor apps and configure application profiles (`sudo ./pmctl run app --data=/opt/pm_db`) -- The _notifier_, a little menu/tray app for quick access and notifications (`sudo ./pmctl run notifier --data=/opt/pm_db`) +**Features/Settings:** -If you want to know more, here are [the docs](http://docs.safing.io/). +- Configure upstream DNS resolvers +- Don't use assigned Nameserver (by DHCP / local network - public WiFi!) +- Don't use Multicast DNS (public WiFi!) -#### Installation +## Privacy Filter -The `pmctl` command will help you get up and running. It will bootstrap your the environment and download additional files it needs. All commands need the `--data` parameter with the database location, as this is where all the data and also the binaries live. +**Status:** _unreleased - pre-alpha scheduled for the next days_ -Just download `pmctl` from the [releases page](https://github.com/safing/portmaster/releases) and put it somewhere comfortable. You may freely choose where you want to put the database - it needs to be the same for all commands. Here we go - run every command in a seperate terminal window: +Think of a pi-hole for your computer. Or an ad-blocker that blocks ads on your whole computer, not only on your browser. With you everywhere you go and every network you visit. -```bash -# Either export the PORTMASTER_DATA environment variable or add -# --data=/opt/pm_db to all commands below. If you use pmctl a -# lot you may move the export line to your ~/.bashrc -export PORTMASTER_DATA=/opt/pm_db +**Features/Settings:** -# start the portmaster: -sudo ./pmctl run core -# this will add some rules to iptables for traffic interception via nfqueue (and will clean up afterwards!) -# already active connections may not be handled correctly, please restart programs for clean behavior +- Select and activate block-lists +- Manually black/whitelist domains + - You can whitelist domains in case something breaks +- CNAME Blocking (block these new nasty "unblockable" ads/trackers - coming soon) +- Block all subdomains of a domain in the block-lists -# then start the app: -./pmctl run app +## Safing Privacy Network (SPN) -# and the notifier: -./pmctl run notifier -``` +**Status:** _unreleased - pre-alpha scheduled for June_ -#### Feedback +Please [visit our Kickstarter campaign](https://www.kickstarter.com/projects/safingio/spn/) to read all about this module. -We'd love to know what you think, drop by on [our forum](https://discourse.safing.community/) and let us know! -If you want to report a bug, please [open an issue on Github](https://github.com/safing/portmaster/issues/new). +# Installation -## Documentation +Installation instructions for your platform as well as known issues can be found at the respective wiki pages: -Documentation _in progress_ can be found here: [docs.safing.io](http://docs.safing.io/) - -## Usage Dependencies - -#### Linux -- libnetfilter_queue - - debian/ubuntu: `sudo apt-get install libnetfilter-queue1` - - fedora: `sudo yum install libnetfilter_queue` - - arch: `sudo pacman -S libnetfilter_queue` -- [Network Manager](https://wiki.gnome.org/Projects/NetworkManager) (_optional_) - -#### Windows -- Windows 7 (with update KB3033929) or up - - [KB3033929](https://docs.microsoft.com/en-us/security-updates/SecurityAdvisories/2015/3033929) (a 2015 security update) is required for correctly verifying the driver signature -- Windows Server 2016 systems must have secure boot disabled. (_clarification needed_) - -## Build Dependencies - -#### Linux -- libnetfilter_queue development files - - debian/ubuntu: `sudo apt-get install libnetfilter-queue-dev` - - fedora: `?` - - arch: `sudo pacman -S libnetfilter_queue` - -## TCP/UDP Ports - -The Portmaster (with Gate17) uses the following ports: -- ` 17` Gate17 port for connecting to Gate17 nodes -- ` 53` DNS server (local only) -- `717` Gate17 entrypoint as the local endpoint for tunneled connections (local only) -- `817` Portmaster API for integration with UI elements and other helpers (local only) - -Learn more about [why we chose these ports](https://docs.safing.io/docs/portmaster/os-integration.html). - -Gate17 nodes additionally uses other common ports like `80` and `443` to provide access in restricted network environments. +- [Linux](https://github.com/safing/portmaster/wiki/Linux) +- [Windows](https://github.com/safing/portmaster/wiki/Windows) diff --git a/core/core.go b/core/core.go index f15a2709..d70569f0 100644 --- a/core/core.go +++ b/core/core.go @@ -18,7 +18,7 @@ var ( ) func init() { - modules.Register("base", nil, registerDatabases, nil, "database", "config", "random") + modules.Register("base", nil, registerDatabases, nil, "database", "config", "rng") module = modules.Register("core", nil, start, nil, "base", "subsystems", "status", "updates", "api", "notifications", "ui") subsystems.Register( diff --git a/firewall/bypassing.go b/firewall/bypassing.go new file mode 100644 index 00000000..ac3349f5 --- /dev/null +++ b/firewall/bypassing.go @@ -0,0 +1,19 @@ +package firewall + +import ( + "strings" + + "github.com/safing/portmaster/network" + "github.com/safing/portmaster/profile/endpoints" +) + +// PreventBypassing checks if the connection should be denied or permitted +// based on some bypass protection checks. +func PreventBypassing(conn *network.Connection) (endpoints.EPResult, string) { + // Block firefox canary domain to disable DoH + if strings.ToLower(conn.Entity.Domain) == "use-application-dns.net." { + return endpoints.Denied, "blocked canary domain to prevent enabling DNS-over-HTTPs" + } + + return endpoints.NoMatch, "" +} diff --git a/firewall/dns.go b/firewall/dns.go new file mode 100644 index 00000000..d9a868cb --- /dev/null +++ b/firewall/dns.go @@ -0,0 +1,230 @@ +package firewall + +import ( + "net" + "os" + "strings" + + "github.com/miekg/dns" + "github.com/safing/portbase/database" + "github.com/safing/portbase/log" + "github.com/safing/portmaster/network" + "github.com/safing/portmaster/network/netutils" + "github.com/safing/portmaster/profile" + "github.com/safing/portmaster/profile/endpoints" + "github.com/safing/portmaster/resolver" +) + +func filterDNSSection(entries []dns.RR, p *profile.LayeredProfile, scope int8) ([]dns.RR, []string, int) { + goodEntries := make([]dns.RR, 0, len(entries)) + filteredRecords := make([]string, 0, len(entries)) + + // keeps track of the number of valid and allowed + // A and AAAA records. + var allowedAddressRecords int + + for _, rr := range entries { + // get IP and classification + var ip net.IP + switch v := rr.(type) { + case *dns.A: + ip = v.A + case *dns.AAAA: + ip = v.AAAA + default: + // add non A/AAAA entries + goodEntries = append(goodEntries, rr) + continue + } + classification := netutils.ClassifyIP(ip) + + if p.RemoveOutOfScopeDNS() { + switch { + case classification == netutils.HostLocal: + // No DNS should return localhost addresses + filteredRecords = append(filteredRecords, rr.String()) + continue + case scope == netutils.Global && (classification == netutils.SiteLocal || classification == netutils.LinkLocal): + // No global DNS should return LAN addresses + filteredRecords = append(filteredRecords, rr.String()) + continue + } + } + + if p.RemoveBlockedDNS() { + // filter by flags + switch { + case p.BlockScopeInternet() && classification == netutils.Global: + filteredRecords = append(filteredRecords, rr.String()) + continue + case p.BlockScopeLAN() && (classification == netutils.SiteLocal || classification == netutils.LinkLocal): + filteredRecords = append(filteredRecords, rr.String()) + continue + case p.BlockScopeLocal() && classification == netutils.HostLocal: + filteredRecords = append(filteredRecords, rr.String()) + continue + } + + // TODO: filter by endpoint list (IP only) + } + + // if survived, add to good entries + allowedAddressRecords++ + goodEntries = append(goodEntries, rr) + } + + return goodEntries, filteredRecords, allowedAddressRecords +} + +func filterDNSResponse(conn *network.Connection, rrCache *resolver.RRCache) *resolver.RRCache { + // do not modify own queries + if conn.Process().Pid == os.Getpid() { + return rrCache + } + + // get profile + p := conn.Process().Profile() + if p == nil { + conn.Block("no profile") + return nil + } + + // check if DNS response filtering is completely turned off + if !p.RemoveOutOfScopeDNS() && !p.RemoveBlockedDNS() { + return rrCache + } + + // duplicate entry + rrCache = rrCache.ShallowCopy() + rrCache.FilteredEntries = make([]string, 0) + + var filteredRecords []string + var validIPs int + + rrCache.Answer, filteredRecords, validIPs = filterDNSSection(rrCache.Answer, p, rrCache.ServerScope) + rrCache.FilteredEntries = append(rrCache.FilteredEntries, filteredRecords...) + + // we don't count the valid IPs in the extra section + rrCache.Extra, filteredRecords, _ = filterDNSSection(rrCache.Extra, p, rrCache.ServerScope) + rrCache.FilteredEntries = append(rrCache.FilteredEntries, filteredRecords...) + + if len(rrCache.FilteredEntries) > 0 { + rrCache.Filtered = true + if validIPs == 0 { + conn.Block("no addresses returned for this domain are permitted") + return nil + } + + log.Infof("filter: filtered DNS replies for %s: %s", conn, strings.Join(rrCache.FilteredEntries, ", ")) + } + + return rrCache +} + +// DecideOnResolvedDNS filters a dns response according to the application profile and settings. +func DecideOnResolvedDNS(conn *network.Connection, q *resolver.Query, rrCache *resolver.RRCache) *resolver.RRCache { + updatedRR := filterDNSResponse(conn, rrCache) + if updatedRR == nil { + return nil + } + + updateIPsAndCNAMEs(q, rrCache, conn) + + if mayBlockCNAMEs(conn) { + return nil + } + + // TODO: Gate17 integration + // tunnelInfo, err := AssignTunnelIP(fqdn) + + return updatedRR +} + +func mayBlockCNAMEs(conn *network.Connection) bool { + // if we have CNAMEs and the profile is configured to filter them + // we need to re-check the lists and endpoints here + if conn.Process().Profile().FilterCNAMEs() { + conn.Entity.ResetLists() + conn.Entity.EnableCNAMECheck(true) + + result, reason := conn.Process().Profile().MatchEndpoint(conn.Entity) + if result == endpoints.Denied { + conn.BlockWithContext(reason.String(), reason.Context()) + return true + } + + if result == endpoints.NoMatch { + result, reason = conn.Process().Profile().MatchFilterLists(conn.Entity) + if result == endpoints.Denied { + conn.BlockWithContext(reason.String(), reason.Context()) + return true + } + } + } + + return false +} + +func updateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *network.Connection) { + // save IP addresses to IPInfo + cnames := make(map[string]string) + ips := make(map[string]struct{}) + + for _, rr := range append(rrCache.Answer, rrCache.Extra...) { + switch v := rr.(type) { + case *dns.CNAME: + cnames[v.Hdr.Name] = v.Target + + case *dns.A: + ips[v.A.String()] = struct{}{} + + case *dns.AAAA: + ips[v.AAAA.String()] = struct{}{} + } + } + + for ip := range ips { + record := resolver.ResolvedDomain{ + Domain: q.FQDN, + } + + // resolve all CNAMEs in the correct order. + var domain = q.FQDN + for { + nextDomain, isCNAME := cnames[domain] + if !isCNAME { + break + } + + record.CNAMEs = append(record.CNAMEs, nextDomain) + domain = nextDomain + } + + // update the entity to include the cnames + conn.Entity.CNAME = record.CNAMEs + + // get the existing IP info or create a new one + var save bool + info, err := resolver.GetIPInfo(ip) + if err != nil { + if err != database.ErrNotFound { + log.Errorf("nameserver: failed to search for IP info record: %s", err) + } + + info = &resolver.IPInfo{ + IP: ip, + } + save = true + } + + // and the new resolved domain record and save + if new := info.AddDomain(record); new { + save = true + } + if save { + if err := info.Save(); err != nil { + log.Errorf("nameserver: failed to save IP info record: %s", err) + } + } + } +} diff --git a/firewall/firewall.go b/firewall/firewall.go index 530f7437..923e87f2 100644 --- a/firewall/firewall.go +++ b/firewall/firewall.go @@ -233,6 +233,7 @@ func initialHandler(conn *network.Connection, pkt packet.Packet) { if ps.isMe { // approve conn.Accept("internally approved") + conn.Internal = true // finish conn.StopFirewallHandler() issueVerdict(conn, pkt, 0, true) diff --git a/firewall/inspection/inspection.go b/firewall/inspection/inspection.go index 55629b19..7dc59494 100644 --- a/firewall/inspection/inspection.go +++ b/firewall/inspection/inspection.go @@ -85,11 +85,11 @@ func RunInspectors(conn *network.Connection, pkt packet.Packet) (network.Verdict verdict = network.VerdictDrop continueInspection = true case BLOCK_CONN: - conn.SetVerdict(network.VerdictBlock) + conn.SetVerdict(network.VerdictBlock, "", nil) verdict = conn.Verdict activeInspectors[key] = true case DROP_CONN: - conn.SetVerdict(network.VerdictDrop) + conn.SetVerdict(network.VerdictDrop, "", nil) verdict = conn.Verdict activeInspectors[key] = true case STOP_INSPECTING: diff --git a/firewall/master.go b/firewall/master.go index 9650b625..06a0f5f1 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -2,7 +2,6 @@ package firewall import ( "fmt" - "net" "os" "path/filepath" "strings" @@ -14,10 +13,8 @@ import ( "github.com/safing/portmaster/process" "github.com/safing/portmaster/profile" "github.com/safing/portmaster/profile/endpoints" - "github.com/safing/portmaster/resolver" "github.com/agext/levenshtein" - "github.com/miekg/dns" ) // Call order: @@ -26,7 +23,7 @@ import ( // 1. DecideOnConnection // is called when a DNS query is made, may set verdict to Undeterminable to permit a DNS reply. // is called with a nil packet. -// 2. FilterDNSResponse +// 2. DecideOnResolvedDNS // is called to (possibly) filter out A/AAAA records that the filter would deny later. // // Network Connection: @@ -35,7 +32,7 @@ import ( // DecideOnConnection makes a decision about a connection. // When called, the connection and profile is already locked. -func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint:gocognit,gocyclo // TODO +func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { // update profiles and check if communication needs reevaluation if conn.UpdateAndCheck() { log.Infof("filter: re-evaluating verdict on %s", conn) @@ -46,13 +43,47 @@ func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint: } } + var deciders = []func(*network.Connection, packet.Packet) bool{ + checkPortmasterConnection, + checkSelfCommunication, + checkProfileExists, + checkConnectionType, + checkConnectionScope, + checkEndpointLists, + checkBypassPrevention, + checkFilterLists, + checkInbound, + checkDefaultPermit, + checkAutoPermitRelated, + checkDefaultAction, + } + + for _, decider := range deciders { + if decider(conn, pkt) { + return + } + } + + // DefaultAction == DefaultActionBlock + conn.Deny("endpoint is not whitelisted (default=block)") +} + +// checkPortmasterConnection allows all connection that originate from +// portmaster itself. +func checkPortmasterConnection(conn *network.Connection, _ packet.Packet) bool { // grant self if conn.Process().Pid == os.Getpid() { log.Infof("filter: granting own connection %s", conn) conn.Verdict = network.VerdictAccept - return + conn.Internal = true + return true } + return false +} + +// checkSelfCommunication checks if the process is communicating with itself. +func checkSelfCommunication(conn *network.Connection, pkt packet.Packet) bool { // check if process is communicating with itself if pkt != nil { // TODO: evaluate the case where different IPs in the 127/8 net are used. @@ -75,18 +106,51 @@ func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint: log.Warningf("filter: failed to find load local peer process with PID %d: %s", otherPid, err) } else if otherProcess.Pid == conn.Process().Pid { conn.Accept("connection to self") - return + conn.Internal = true + return true } } } } - // get profile - p := conn.Process().Profile() - if p == nil { - conn.Block("no profile") - return + return false +} + +func checkProfileExists(conn *network.Connection, _ packet.Packet) bool { + if conn.Process().Profile() == nil { + conn.Block("unknown process or profile") + return true } + return false +} + +func checkEndpointLists(conn *network.Connection, _ packet.Packet) bool { + var result endpoints.EPResult + var reason endpoints.Reason + + // there must always be a profile. + p := conn.Process().Profile() + + // check endpoints list + if conn.Inbound { + result, reason = p.MatchServiceEndpoint(conn.Entity) + } else { + result, reason = p.MatchEndpoint(conn.Entity) + } + switch result { + case endpoints.Denied: + conn.DenyWithContext(reason.String(), reason.Context()) + return true + case endpoints.Permitted: + conn.AcceptWithContext(reason.String(), reason.Context()) + return true + } + + return false +} + +func checkConnectionType(conn *network.Connection, _ packet.Packet) bool { + p := conn.Process().Profile() // check conn type switch conn.Scope { @@ -97,16 +161,22 @@ func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint: } else { conn.Drop("inbound connections blocked") } - return + return true } case network.PeerLAN, network.PeerInternet, network.PeerInvalid: // Important: PeerHost is and should be missing! if p.BlockP2P() { conn.Block("direct connections (P2P) blocked") - return + return true } } + return false +} + +func checkConnectionScope(conn *network.Connection, _ packet.Packet) bool { + p := conn.Process().Profile() + // check scopes if conn.Entity.IP != nil { classification := netutils.ClassifyIP(conn.Entity.IP) @@ -115,21 +185,21 @@ func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint: case netutils.Global, netutils.GlobalMulticast: if p.BlockScopeInternet() { conn.Deny("Internet access blocked") // Block Outbound / Drop Inbound - return + return true } case netutils.SiteLocal, netutils.LinkLocal, netutils.LocalMulticast: if p.BlockScopeLAN() { conn.Block("LAN access blocked") // Block Outbound / Drop Inbound - return + return true } case netutils.HostLocal: if p.BlockScopeLocal() { conn.Block("Localhost access blocked") // Block Outbound / Drop Inbound - return + return true } default: // netutils.Invalid conn.Deny("invalid IP") // Block Outbound / Drop Inbound - return + return true } } else if conn.Entity.Domain != "" { // DNS Query @@ -137,182 +207,84 @@ func DecideOnConnection(conn *network.Connection, pkt packet.Packet) { //nolint: // TODO: handle domains mapped to localhost if p.BlockScopeInternet() && p.BlockScopeLAN() { conn.Block("Internet and LAN access blocked") - return + return true } } + return false +} - // check endpoints list - var result endpoints.EPResult - var reason string - if conn.Inbound { - result, reason = p.MatchServiceEndpoint(conn.Entity) - } else { - result, reason = p.MatchEndpoint(conn.Entity) +func checkBypassPrevention(conn *network.Connection, _ packet.Packet) bool { + if conn.Process().Profile().PreventBypassing() { + // check for bypass protection + result, reason := PreventBypassing(conn) + switch result { + case endpoints.Denied: + conn.Block("bypass prevention: " + reason) + return true + case endpoints.Permitted: + conn.Accept("bypass prevention: " + reason) + return true + case endpoints.NoMatch: + } } - switch result { - case endpoints.Denied: - conn.Deny("endpoint is blacklisted: " + reason) // Block Outbound / Drop Inbound - return - case endpoints.Permitted: - conn.Accept("endpoint is whitelisted: " + reason) - return - } - // continuing with result == NoMatch + return false +} +func checkFilterLists(conn *network.Connection, _ packet.Packet) bool { // apply privacy filter lists - result, reason = p.MatchFilterLists(conn.Entity) + p := conn.Process().Profile() + + result, reason := p.MatchFilterLists(conn.Entity) switch result { case endpoints.Denied: - conn.Deny("endpoint in filterlists: " + reason) - return + conn.DenyWithContext(reason.String(), reason.Context()) + return true case endpoints.NoMatch: // nothing to do default: log.Debugf("filter: filter lists returned unsupported verdict: %s", result) } + return false +} +func checkInbound(conn *network.Connection, _ packet.Packet) bool { // implicit default=block for inbound if conn.Inbound { conn.Drop("endpoint is not whitelisted (incoming is always default=block)") - return + return true } + return false +} +func checkDefaultPermit(conn *network.Connection, _ packet.Packet) bool { // check default action + p := conn.Process().Profile() if p.DefaultAction() == profile.DefaultActionPermit { conn.Accept("endpoint is not blacklisted (default=permit)") - return + return true } + return false +} - // check relation +func checkAutoPermitRelated(conn *network.Connection, _ packet.Packet) bool { + p := conn.Process().Profile() if !p.DisableAutoPermit() { related, reason := checkRelation(conn) if related { conn.Accept(reason) - return + return true } } - - // prompt - if p.DefaultAction() == profile.DefaultActionAsk { - prompt(conn, pkt) - return - } - - // DefaultAction == DefaultActionBlock - conn.Deny("endpoint is not whitelisted (default=block)") + return false } -// FilterDNSResponse filters a dns response according to the application profile and settings. -func FilterDNSResponse(conn *network.Connection, q *resolver.Query, rrCache *resolver.RRCache) *resolver.RRCache { //nolint:gocognit // TODO - // do not modify own queries - if conn.Process().Pid == os.Getpid() { - return rrCache - } - - // get profile +func checkDefaultAction(conn *network.Connection, pkt packet.Packet) bool { p := conn.Process().Profile() - if p == nil { - conn.Block("no profile") - return nil + if p.DefaultAction() == profile.DefaultActionAsk { + prompt(conn, pkt) + return true } - - // check if DNS response filtering is completely turned off - if !p.RemoveOutOfScopeDNS() && !p.RemoveBlockedDNS() { - return rrCache - } - - // duplicate entry - rrCache = rrCache.ShallowCopy() - rrCache.FilteredEntries = make([]string, 0) - - // change information - var addressesRemoved int - var addressesOk int - - // loop vars - var classification int8 - var ip net.IP - - // filter function - filterEntries := func(entries []dns.RR) (goodEntries []dns.RR) { - goodEntries = make([]dns.RR, 0, len(entries)) - - for _, rr := range entries { - - // get IP and classification - switch v := rr.(type) { - case *dns.A: - ip = v.A - case *dns.AAAA: - ip = v.AAAA - default: - // add non A/AAAA entries - goodEntries = append(goodEntries, rr) - continue - } - classification = netutils.ClassifyIP(ip) - - if p.RemoveOutOfScopeDNS() { - switch { - case classification == netutils.HostLocal: - // No DNS should return localhost addresses - addressesRemoved++ - rrCache.FilteredEntries = append(rrCache.FilteredEntries, rr.String()) - continue - case rrCache.ServerScope == netutils.Global && (classification == netutils.SiteLocal || classification == netutils.LinkLocal): - // No global DNS should return LAN addresses - addressesRemoved++ - rrCache.FilteredEntries = append(rrCache.FilteredEntries, rr.String()) - continue - } - } - - if p.RemoveBlockedDNS() { - // filter by flags - switch { - case p.BlockScopeInternet() && classification == netutils.Global: - addressesRemoved++ - rrCache.FilteredEntries = append(rrCache.FilteredEntries, rr.String()) - continue - case p.BlockScopeLAN() && (classification == netutils.SiteLocal || classification == netutils.LinkLocal): - addressesRemoved++ - rrCache.FilteredEntries = append(rrCache.FilteredEntries, rr.String()) - continue - case p.BlockScopeLocal() && classification == netutils.HostLocal: - addressesRemoved++ - rrCache.FilteredEntries = append(rrCache.FilteredEntries, rr.String()) - continue - } - - // TODO: filter by endpoint list (IP only) - } - - // if survived, add to good entries - addressesOk++ - goodEntries = append(goodEntries, rr) - } - return - } - - rrCache.Answer = filterEntries(rrCache.Answer) - rrCache.Extra = filterEntries(rrCache.Extra) - - if addressesRemoved > 0 { - rrCache.Filtered = true - if addressesOk == 0 { - conn.Block("no addresses returned for this domain are permitted") - return nil - } - } - - if rrCache.Filtered { - log.Infof("filter: filtered DNS replies for %s: %s", conn, strings.Join(rrCache.FilteredEntries, ", ")) - } - - // TODO: Gate17 integration - // tunnelInfo, err := AssignTunnelIP(fqdn) - - return rrCache + return false } // checkRelation tries to find a relation between a process and a communication. This is for better out of the box experience and is _not_ meant to thwart intentional malware. diff --git a/intel/block_reason.go b/intel/block_reason.go new file mode 100644 index 00000000..09b89db2 --- /dev/null +++ b/intel/block_reason.go @@ -0,0 +1,97 @@ +package intel + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/miekg/dns" + "github.com/safing/portbase/log" +) + +// ListMatch represents an entity that has been +// matched against filterlists. +type ListMatch struct { + Entity string + ActiveLists []string + InactiveLists []string +} + +func (lm *ListMatch) String() string { + inactive := "" + if len(lm.InactiveLists) > 0 { + inactive = " and in deactivated lists " + strings.Join(lm.InactiveLists, ", ") + } + return fmt.Sprintf( + "%s in activated lists %s%s", + lm.Entity, + strings.Join(lm.ActiveLists, ","), + inactive, + ) +} + +// ListBlockReason is a list of list matches. +type ListBlockReason []ListMatch + +func (br ListBlockReason) String() string { + if len(br) == 0 { + return "" + } + + matches := make([]string, len(br)) + for idx, lm := range br { + matches[idx] = lm.String() + } + + return strings.Join(matches, " and ") +} + +// Context returns br wrapped into a map. It implements +// the endpoints.Reason interface. +func (br ListBlockReason) Context() interface{} { + return br +} + +// MarshalJSON marshals the list block reason into a map +// prefixed with filterlists. +func (br ListBlockReason) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + // we convert to []ListMatch to avoid recursing + // here. + "filterlists": []ListMatch(br), + }) +} + +// ToRRs returns a set of dns TXT records that describe the +// block reason. +func (br ListBlockReason) ToRRs() []dns.RR { + rrs := make([]dns.RR, 0, len(br)) + + for _, lm := range br { + blockedBy, err := dns.NewRR(fmt.Sprintf( + "%s-blockedBy. 0 IN TXT %q", + strings.TrimRight(lm.Entity, "."), + strings.Join(lm.ActiveLists, ","), + )) + if err == nil { + rrs = append(rrs, blockedBy) + } else { + log.Errorf("intel: failed to create TXT RR for block reason: %s", err) + } + + if len(lm.InactiveLists) > 0 { + wouldBeBlockedBy, err := dns.NewRR(fmt.Sprintf( + "%s-wouldBeBlockedBy. 0 IN TXT %q", + strings.TrimRight(lm.Entity, "."), + strings.Join(lm.InactiveLists, ","), + )) + if err == nil { + rrs = append(rrs, wouldBeBlockedBy) + } else { + log.Errorf("intel: failed to create TXT RR for block reason: %s", err) + } + } + } + + return rrs +} diff --git a/intel/entity.go b/intel/entity.go index f4412bc3..af96343d 100644 --- a/intel/entity.go +++ b/intel/entity.go @@ -32,18 +32,46 @@ type Entity struct { asnListLoaded bool reverseResolveEnabled bool resolveSubDomainLists bool + checkCNAMEs bool + // Protocol is the protcol number used by the connection. Protocol uint8 - Port uint16 - Domain string - IP net.IP - Country string - ASN uint + // Port is the destination port of the connection + Port uint16 + + // Domain is the target domain of the connection. + Domain string + + // CNAME is a list of domain names that have been + // resolved for Domain. + CNAME []string + + // IP is the IP address of the connection. If domain is + // set, IP has been resolved by following all CNAMEs. + IP net.IP + + // Country holds the country the IP address (ASN) is + // located in. + Country string + + // ASN holds the autonomous system number of the IP. + ASN uint + location *geoip.Location - Lists []string - ListsMap filterlists.LookupMap + // BlockedByLists holds list source IDs that + // are used to block the entity. + BlockedByLists []string + + // BlockedEntities holds a list of entities that + // have been blocked. Values can be used as a key + // for the ListOccurences map. + BlockedEntities []string + + // ListOccurences is a map that matches an entity (Domain, IPs, ASN, Country, Sub-domain) + // to a list of sources where the entity has been observed in. + ListOccurences map[string][]string // we only load each data above at most once fetchLocationOnce sync.Once @@ -72,13 +100,17 @@ func (e *Entity) ResetLists() { // TODO(ppacher): our actual goal is to reset the domain // list right now so we could be more efficient by keeping // the other lists around. - e.Lists = nil - e.ListsMap = nil + + e.BlockedByLists = nil + e.BlockedEntities = nil + e.ListOccurences = nil + e.domainListLoaded = false e.ipListLoaded = false e.countryListLoaded = false e.asnListLoaded = false e.resolveSubDomainLists = false + e.checkCNAMEs = false e.loadDomainListOnce = sync.Once{} e.loadIPListOnce = sync.Once{} e.loadCoutryListOnce = sync.Once{} @@ -94,6 +126,21 @@ func (e *Entity) ResolveSubDomainLists(enabled bool) { e.resolveSubDomainLists = enabled } +// EnableCNAMECheck enalbes or disables list lookups for +// entity CNAMEs. +func (e *Entity) EnableCNAMECheck(enabled bool) { + if e.domainListLoaded { + log.Warningf("intel/filterlists: tried to change CNAME resolving for %s but lists are already fetched", e.Domain) + } + e.checkCNAMEs = enabled +} + +// CNAMECheckEnabled returns true if the entities CNAMEs should +// also be checked. +func (e *Entity) CNAMECheckEnabled() bool { + return e.checkCNAMEs +} + // Domain and IP // EnableReverseResolving enables reverse resolving the domain from the IP on demand. @@ -151,7 +198,6 @@ func (e *Entity) getLocation() { e.fetchLocationOnce.Do(func() { // need IP! if e.IP == nil { - log.Warningf("intel: cannot get location for %s data without IP", e.Domain) return } @@ -205,9 +251,19 @@ func (e *Entity) getLists() { e.getCountryLists() } -func (e *Entity) mergeList(list []string) { - e.Lists = mergeStringList(e.Lists, list) - e.ListsMap = buildLookupMap(e.Lists) +func (e *Entity) mergeList(key string, list []string) { + if len(list) == 0 { + return + } + + if e.ListOccurences == nil { + e.ListOccurences = make(map[string][]string) + } + + e.ListOccurences[key] = mergeStringList(e.ListOccurences[key], list) + + //e.Lists = mergeStringList(e.Lists, list) + //e.ListsMap = buildLookupMap(e.Lists) } func (e *Entity) getDomainLists() { @@ -221,14 +277,29 @@ func (e *Entity) getDomainLists() { } e.loadDomainListOnce.Do(func() { - var domains = []string{domain} - if e.resolveSubDomainLists { - domains = splitDomain(domain) - log.Debugf("intel: subdomain list resolving is enabled, checking %v", domains) + var domainsToInspect = []string{domain} + + if e.checkCNAMEs { + log.Tracef("intel: CNAME filtering enabled, checking %v too", e.CNAME) + domainsToInspect = append(domainsToInspect, e.CNAME...) } + var domains []string + if e.resolveSubDomainLists { + for _, domain := range domainsToInspect { + subdomains := splitDomain(domain) + domains = append(domains, subdomains...) + + log.Tracef("intel: subdomain list resolving is enabled: %s => %v", domains, subdomains) + } + } else { + domains = domainsToInspect + } + + domains = makeDistinct(domains) + for _, d := range domains { - log.Debugf("intel: loading domain list for %s", d) + log.Tracef("intel: loading domain list for %s", d) list, err := filterlists.LookupDomain(d) if err != nil { log.Errorf("intel: failed to get domain blocklists for %s: %s", d, err) @@ -236,7 +307,7 @@ func (e *Entity) getDomainLists() { return } - e.mergeList(list) + e.mergeList(d, list) } e.domainListLoaded = true }) @@ -278,9 +349,10 @@ func (e *Entity) getASNLists() { return } - log.Debugf("intel: loading ASN list for %d", asn) + log.Tracef("intel: loading ASN list for %d", asn) e.loadAsnListOnce.Do(func() { - list, err := filterlists.LookupASNString(fmt.Sprintf("%d", asn)) + asnStr := fmt.Sprintf("%d", asn) + list, err := filterlists.LookupASNString(asnStr) if err != nil { log.Errorf("intel: failed to get ASN blocklist for %d: %s", asn, err) e.loadAsnListOnce = sync.Once{} @@ -288,7 +360,7 @@ func (e *Entity) getASNLists() { } e.asnListLoaded = true - e.mergeList(list) + e.mergeList(asnStr, list) }) } @@ -302,7 +374,7 @@ func (e *Entity) getCountryLists() { return } - log.Debugf("intel: loading country list for %s", country) + log.Tracef("intel: loading country list for %s", country) e.loadCoutryListOnce.Do(func() { list, err := filterlists.LookupCountry(country) if err != nil { @@ -312,7 +384,7 @@ func (e *Entity) getCountryLists() { } e.countryListLoaded = true - e.mergeList(list) + e.mergeList(country, list) }) } @@ -335,7 +407,7 @@ func (e *Entity) getIPLists() { return } - log.Debugf("intel: loading IP list for %s", ip) + log.Tracef("intel: loading IP list for %s", ip) e.loadIPListOnce.Do(func() { list, err := filterlists.LookupIP(ip) @@ -345,28 +417,69 @@ func (e *Entity) getIPLists() { return } e.ipListLoaded = true - e.mergeList(list) + e.mergeList(ip.String(), list) }) } -// GetLists returns the filter list identifiers the entity matched and whether this data is set. -func (e *Entity) GetLists() ([]string, bool) { +// LoadLists searches all filterlists for all occurrences of +// this entity. +func (e *Entity) LoadLists() bool { e.getLists() - if e.Lists == nil { - return nil, false - } - return e.Lists, true + return e.ListOccurences != nil } -// GetListsMap is like GetLists but returns a lookup map for list IDs. -func (e *Entity) GetListsMap() (filterlists.LookupMap, bool) { - e.getLists() +// MatchLists matches the entities lists against a slice +// of source IDs and updates various entity properties +// like BlockedByLists, ListOccurences and BlockedEntitites. +func (e *Entity) MatchLists(lists []string) bool { + e.BlockedByLists = nil + e.BlockedEntities = nil - if e.ListsMap == nil { - return nil, false + lm := makeMap(lists) + for key, keyLists := range e.ListOccurences { + for _, keyListID := range keyLists { + if _, ok := lm[keyListID]; ok { + e.BlockedByLists = append(e.BlockedByLists, keyListID) + e.BlockedEntities = append(e.BlockedEntities, key) + } + } } - return e.ListsMap, true + + makeDistinct(e.BlockedByLists) + makeDistinct(e.BlockedEntities) + + return len(e.BlockedByLists) > 0 +} + +// ListBlockReason returns the block reason for this entity. +func (e *Entity) ListBlockReason() ListBlockReason { + blockedBy := make([]ListMatch, len(e.BlockedEntities)) + + lm := makeMap(e.BlockedByLists) + + for idx, blockedEntity := range e.BlockedEntities { + if entityLists, ok := e.ListOccurences[blockedEntity]; ok { + var activeLists []string + var inactiveLists []string + + for _, l := range entityLists { + if _, ok := lm[l]; ok { + activeLists = append(activeLists, l) + } else { + inactiveLists = append(inactiveLists, l) + } + } + + blockedBy[idx] = ListMatch{ + Entity: blockedEntity, + ActiveLists: activeLists, + InactiveLists: inactiveLists, + } + } + } + + return blockedBy } func mergeStringList(a, b []string) []string { @@ -386,12 +499,26 @@ func mergeStringList(a, b []string) []string { return res } -func buildLookupMap(l []string) filterlists.LookupMap { - m := make(filterlists.LookupMap, len(l)) +func makeDistinct(slice []string) []string { + m := make(map[string]struct{}, len(slice)) + result := make([]string, 0, len(slice)) - for _, s := range l { - m[s] = struct{}{} + for _, v := range slice { + if _, ok := m[v]; ok { + continue + } + + m[v] = struct{}{} + result = append(result, v) } - return m + return result +} + +func makeMap(slice []string) map[string]struct{} { + lm := make(map[string]struct{}) + for _, v := range slice { + lm[v] = struct{}{} + } + return lm } diff --git a/intel/filterlists/lookup_map.go b/intel/filterlists/lookup_map.go deleted file mode 100644 index 5a161c91..00000000 --- a/intel/filterlists/lookup_map.go +++ /dev/null @@ -1,25 +0,0 @@ -package filterlists - -import "strings" - -// LookupMap is a helper type for matching a list of endpoint sources -// against a map. -type LookupMap map[string]struct{} - -// Match checks if a source in `list` is part of lm. -// Matches are joined to string and returned. -// If nothing is found, an empty string is returned. -func (lm LookupMap) Match(list []string) string { - matches := make([]string, 0, len(list)) - for _, l := range list { - if _, ok := lm[l]; ok { - matches = append(matches, l) - } - } - - if len(matches) == 0 { - return "" - } - - return strings.Join(matches, ", ") -} diff --git a/intel/filterlists/lookup_test.go b/intel/filterlists/lookup_test.go deleted file mode 100644 index 0a2e9220..00000000 --- a/intel/filterlists/lookup_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package filterlists - -/* - -func TestLookupASN(t *testing.T) { - lists, err := LookupASNString("123") - assert.NoError(t, err) - assert.Equal(t, []string{"TEST"}, lists) - - lists, err = LookupASNString("does-not-exist") - assert.NoError(t, err) - assert.Empty(t, lists) - - defer testMarkNotLoaded()() - lists, err = LookupASNString("123") - assert.NoError(t, err) - assert.Empty(t, lists) -} - -func TestLookupCountry(t *testing.T) { - lists, err := LookupCountry("AT") - assert.NoError(t, err) - assert.Equal(t, []string{"TEST"}, lists) - - lists, err = LookupCountry("does-not-exist") - assert.NoError(t, err) - assert.Empty(t, lists) - - defer testMarkNotLoaded()() - lists, err = LookupCountry("AT") - assert.NoError(t, err) - assert.Empty(t, lists) -} - -func TestLookupIP(t *testing.T) { - lists, err := LookupIP(net.IP{1, 1, 1, 1}) - assert.NoError(t, err) - assert.Equal(t, []string{"TEST"}, lists) - - lists, err = LookupIP(net.IP{127, 0, 0, 1}) - assert.NoError(t, err) - assert.Empty(t, lists) - - defer testMarkNotLoaded()() - lists, err = LookupIP(net.IP{1, 1, 1, 1}) - assert.NoError(t, err) - assert.Empty(t, lists) -} - -func TestLookupDomain(t *testing.T) { - lists, err := LookupDomain("example.com") - assert.NoError(t, err) - assert.Equal(t, []string{"TEST"}, lists) - - lists, err = LookupDomain("does-not-exist") - assert.NoError(t, err) - assert.Empty(t, lists) - - defer testMarkNotLoaded()() - lists, err = LookupDomain("example.com") - assert.NoError(t, err) - assert.Empty(t, lists) -} - -// testMarkNotLoaded ensures that functions believe -// filterlists are not yet loaded. It returns a -// func that restores the previous state. -func testMarkNotLoaded() func() { - if isLoaded() { - filterListsLoaded = make(chan struct{}) - return func() { - close(filterListsLoaded) - } - } - - return func() {} -} - -// testMarkLoaded is like testMarkNotLoaded but ensures -// isLoaded() return true. It returns a function to restore -// the previous state. -func testMarkLoaded() func() { - if !isLoaded() { - close(filterListsLoaded) - return func() { - filterListsLoaded = make(chan struct{}) - } - } - - return func() {} -} -*/ diff --git a/intel/lists.go b/intel/lists.go deleted file mode 100644 index 08ca0925..00000000 --- a/intel/lists.go +++ /dev/null @@ -1,40 +0,0 @@ -package intel - -// ListSet holds a set of list IDs. -type ListSet struct { - match []string -} - -// NewListSet returns a new ListSet with the given list IDs. -func NewListSet(lists []string) *ListSet { - // TODO: validate lists - return &ListSet{ - match: lists, - } -} - -// Matches returns whether there is a match in the given list IDs. -func (ls *ListSet) Matches(lists []string) (matches bool) { - for _, list := range lists { - for _, entry := range ls.match { - if entry == list { - return true - } - } - } - - return false -} - -// MatchSet returns the matching list IDs. -func (ls *ListSet) MatchSet(lists []string) (matched []string) { - for _, list := range lists { - for _, entry := range ls.match { - if entry == list { - matched = append(matched, list) - } - } - } - - return -} diff --git a/main.go b/main.go index b45f5ddf..520e8a2d 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,6 @@ import ( ) func main() { - info.Set("Portmaster", "0.4.0", "AGPLv3", true) + info.Set("Portmaster", "0.4.1", "AGPLv3", true) os.Exit(run.Run()) } diff --git a/nameserver/nameserver.go b/nameserver/nameserver.go index 7eee0c4d..03d71701 100644 --- a/nameserver/nameserver.go +++ b/nameserver/nameserver.go @@ -2,6 +2,8 @@ package nameserver import ( "context" + "errors" + "fmt" "net" "strings" @@ -87,10 +89,27 @@ func stop() error { return nil } -func returnNXDomain(w dns.ResponseWriter, query *dns.Msg) { +func returnNXDomain(w dns.ResponseWriter, query *dns.Msg, reason string, reasonContext interface{}) { m := new(dns.Msg) m.SetRcode(query, dns.RcodeNameError) - _ = w.WriteMsg(m) + rr, _ := dns.NewRR("portmaster.block-reason. 0 IN TXT " + fmt.Sprintf("%q", reason)) + m.Extra = []dns.RR{rr} + + if reasonContext != nil { + if v, ok := reasonContext.(interface { + ToRRs() []dns.RR + }); ok { + m.Extra = append(m.Extra, v.ToRRs()...) + } else if v, ok := reasonContext.(interface { + ToRR() dns.RR + }); ok { + m.Extra = append(m.Extra, v.ToRR()) + } + } + + if err := w.WriteMsg(m); err != nil { + log.Errorf("nameserver: failed to send response: %s", err) + } } func returnServerFailure(w dns.ResponseWriter, query *dns.Msg) { @@ -126,7 +145,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er if question.Qclass != dns.ClassINET { // we only serve IN records, return nxdomain log.Warningf("nameserver: only IN record requests are supported but received Qclass %d, returning NXDOMAIN", question.Qclass) - returnNXDomain(w, query) + returnNXDomain(w, query, "wrong type", nil) return nil } @@ -166,7 +185,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er // check if valid domain name if !netutils.IsValidFqdn(q.FQDN) { log.Debugf("nameserver: domain name %s is invalid, returning nxdomain", q.FQDN) - returnNXDomain(w, query) + returnNXDomain(w, query, "invalid domain", nil) return nil } @@ -177,7 +196,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er // TODO: if there are 3 request for the same domain/type in a row, delete all caches of that domain // get connection - conn := network.NewConnectionFromDNSRequest(ctx, q.FQDN, remoteAddr.IP, uint16(remoteAddr.Port)) + conn := network.NewConnectionFromDNSRequest(ctx, q.FQDN, nil, remoteAddr.IP, uint16(remoteAddr.Port)) // once we decided on the connection we might need to save it to the database // so we defer that check right now. @@ -199,12 +218,13 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er } }() + // TODO: this has been obsoleted due to special profiles if conn.Process().Profile() == nil { tracer.Infof("nameserver: failed to find process for request %s, returning NXDOMAIN", conn) - returnNXDomain(w, query) // NOTE(ppacher): saving unknown process connection might end up in a lot of // processes. Consider disabling that via config. conn.Failed("Unknown process") + returnNXDomain(w, query, "unknown process", conn.ReasonContext) return nil } @@ -217,8 +237,8 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er // log.Tracef("nameserver: domain %s has lms score of %f", fqdn, lms) if lms < 10 { tracer.Warningf("nameserver: possible data tunnel by %s: %s has lms score of %f, returning nxdomain", conn.Process(), q.FQDN, lms) - returnNXDomain(w, query) conn.Block("Possible data tunnel") + returnNXDomain(w, query, "lms", conn.ReasonContext) return nil } @@ -228,7 +248,7 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er switch conn.Verdict { case network.VerdictBlock: tracer.Infof("nameserver: %s blocked, returning nxdomain", conn) - returnNXDomain(w, query) + returnNXDomain(w, query, conn.Reason, conn.ReasonContext) return nil case network.VerdictDrop, network.VerdictFailed: tracer.Infof("nameserver: %s dropped, not replying", conn) @@ -240,53 +260,21 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er if err != nil { // TODO: analyze nxdomain requests, malware could be trying DGA-domains tracer.Warningf("nameserver: %s requested %s%s: %s", conn.Process(), q.FQDN, q.QType, err) - returnNXDomain(w, query) - conn.Failed("failed to resolve: " + err.Error()) - return nil - } - // filter DNS response - rrCache = firewall.FilterDNSResponse(conn, q, rrCache) - // TODO: FilterDNSResponse also sets a connection verdict - if rrCache == nil { - tracer.Infof("nameserver: %s implicitly denied by filtering the dns response, returning nxdomain", conn) - returnNXDomain(w, query) - conn.Block("DNS response filtered") - return nil - } - - // save IP addresses to IPInfo - for _, rr := range append(rrCache.Answer, rrCache.Extra...) { - switch v := rr.(type) { - case *dns.A: - ipInfo, err := resolver.GetIPInfo(v.A.String()) - if err != nil { - ipInfo = &resolver.IPInfo{ - IP: v.A.String(), - Domains: []string{q.FQDN}, - } - _ = ipInfo.Save() - } else { - added := ipInfo.AddDomain(q.FQDN) - if added { - _ = ipInfo.Save() - } - } - case *dns.AAAA: - ipInfo, err := resolver.GetIPInfo(v.AAAA.String()) - if err != nil { - ipInfo = &resolver.IPInfo{ - IP: v.AAAA.String(), - Domains: []string{q.FQDN}, - } - _ = ipInfo.Save() - } else { - added := ipInfo.AddDomain(q.FQDN) - if added { - _ = ipInfo.Save() - } - } + if errors.Is(err, &resolver.BlockedUpstreamError{}) { + conn.Block(err.Error()) + } else { + conn.Failed("failed to resolve: " + err.Error()) } + + returnNXDomain(w, query, conn.Reason, conn.ReasonContext) + return nil + } + + rrCache = firewall.DecideOnResolvedDNS(conn, q, rrCache) + if rrCache == nil { + returnNXDomain(w, query, conn.Reason, conn.ReasonContext) + return nil } // reply to query diff --git a/nameserver/only/nameserver.go b/nameserver/only/nameserver.go index feaa5ca5..6d5cb5cb 100644 --- a/nameserver/only/nameserver.go +++ b/nameserver/only/nameserver.go @@ -5,6 +5,7 @@ import ( "net" "strings" + "github.com/safing/portbase/database" "github.com/safing/portbase/log" "github.com/safing/portbase/modules" "github.com/safing/portmaster/netenv" @@ -164,35 +165,60 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er } // save IP addresses to IPInfo + cnames := make(map[string]string) + ips := make(map[string]struct{}) + for _, rr := range append(rrCache.Answer, rrCache.Extra...) { switch v := rr.(type) { + case *dns.CNAME: + cnames[v.Hdr.Name] = v.Target + case *dns.A: - ipInfo, err := resolver.GetIPInfo(v.A.String()) - if err != nil { - ipInfo = &resolver.IPInfo{ - IP: v.A.String(), - Domains: []string{q.FQDN}, - } - _ = ipInfo.Save() - } else { - added := ipInfo.AddDomain(q.FQDN) - if added { - _ = ipInfo.Save() - } - } + ips[v.A.String()] = struct{}{} + case *dns.AAAA: - ipInfo, err := resolver.GetIPInfo(v.AAAA.String()) - if err != nil { - ipInfo = &resolver.IPInfo{ - IP: v.AAAA.String(), - Domains: []string{q.FQDN}, - } - _ = ipInfo.Save() - } else { - added := ipInfo.AddDomain(q.FQDN) - if added { - _ = ipInfo.Save() - } + ips[v.AAAA.String()] = struct{}{} + } + } + + for ip := range ips { + record := resolver.ResolvedDomain{ + Domain: q.FQDN, + } + + // resolve all CNAMEs in the correct order. + var domain = q.FQDN + for { + nextDomain, isCNAME := cnames[domain] + if !isCNAME { + break + } + + record.CNAMEs = append(record.CNAMEs, nextDomain) + domain = nextDomain + } + + // get the existing IP info or create a new one + var save bool + info, err := resolver.GetIPInfo(ip) + if err != nil { + if err != database.ErrNotFound { + log.Errorf("nameserver: failed to search for IP info record: %s", err) + } + + info = &resolver.IPInfo{ + IP: ip, + } + save = true + } + + // and the new resolved domain record and save + if new := info.AddDomain(record); new { + save = true + } + if save { + if err := info.Save(); err != nil { + log.Errorf("nameserver: failed to save IP info record: %s", err) } } } diff --git a/network/clean.go b/network/clean.go index 2c8cff02..ec51b611 100644 --- a/network/clean.go +++ b/network/clean.go @@ -57,8 +57,7 @@ func cleanConnections() (activePIDs map[int]struct{}) { // Step 2: mark end activePIDs[conn.process.Pid] = struct{}{} conn.Ended = now - // "save" - dbController.PushUpdate(conn) + conn.Save() } case conn.Ended < deleteOlderThan: // Step 3: delete diff --git a/network/connection.go b/network/connection.go index d2ada322..bbc088dc 100644 --- a/network/connection.go +++ b/network/connection.go @@ -31,9 +31,10 @@ type Connection struct { //nolint:maligned // TODO: fix alignment Entity *intel.Entity // needs locking, instance is never shared process *process.Process - Verdict Verdict - Reason string - ReasonID string // format source[:id[:id]] // TODO + Verdict Verdict + Reason string + ReasonContext interface{} + ReasonID string // format source[:id[:id]] // TODO Started int64 Ended int64 @@ -41,6 +42,7 @@ type Connection struct { //nolint:maligned // TODO: fix alignment VerdictPermanent bool Inspecting bool Encrypted bool // TODO + Internal bool // Portmaster internal connections are marked in order to easily filter these out in the UI pktQueue chan packet.Packet firewallHandler FirewallHandler @@ -53,12 +55,12 @@ type Connection struct { //nolint:maligned // TODO: fix alignment } // NewConnectionFromDNSRequest returns a new connection based on the given dns request. -func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, ip net.IP, port uint16) *Connection { +func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, cnames []string, localIP net.IP, localPort uint16) *Connection { // get Process - proc, err := process.GetProcessByEndpoints(ctx, ip, port, dnsAddress, dnsPort, packet.UDP) + proc, err := process.GetProcessByEndpoints(ctx, localIP, localPort, dnsAddress, dnsPort, packet.UDP) if err != nil { log.Warningf("network: failed to find process of dns request for %s: %s", fqdn, err) - proc = process.UnknownProcess + proc = process.GetUnidentifiedProcess(ctx) } timestamp := time.Now().Unix() @@ -66,7 +68,8 @@ func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, ip net.IP, po Scope: fqdn, Entity: (&intel.Entity{ Domain: fqdn, - }).Init(), + CNAME: cnames, + }), process: proc, Started: timestamp, Ended: timestamp, @@ -80,7 +83,7 @@ func NewConnectionFromFirstPacket(pkt packet.Packet) *Connection { proc, inbound, err := process.GetProcessByPacket(pkt) if err != nil { log.Warningf("network: failed to find process of packet %s: %s", pkt, err) - proc = process.UnknownProcess + proc = process.GetUnidentifiedProcess(pkt.Ctx()) } var scope string @@ -103,7 +106,7 @@ func NewConnectionFromFirstPacket(pkt packet.Packet) *Connection { IP: pkt.Info().Src, Protocol: uint8(pkt.Info().Protocol), Port: pkt.Info().SrcPort, - }).Init() + }) } else { @@ -112,18 +115,21 @@ func NewConnectionFromFirstPacket(pkt packet.Packet) *Connection { IP: pkt.Info().Dst, Protocol: uint8(pkt.Info().Protocol), Port: pkt.Info().DstPort, - }).Init() + }) // check if we can find a domain for that IP ipinfo, err := resolver.GetIPInfo(pkt.Info().Dst.String()) if err == nil { + lastResolvedDomain := ipinfo.ResolvedDomains.MostRecentDomain() + if lastResolvedDomain != nil { + scope = lastResolvedDomain.Domain + entity.Domain = lastResolvedDomain.Domain + entity.CNAME = lastResolvedDomain.CNAMEs + removeOpenDNSRequest(proc.Pid, lastResolvedDomain.Domain) + } + } - // outbound to domain - scope = ipinfo.Domains[0] - entity.Domain = scope - removeOpenDNSRequest(proc.Pid, scope) - - } else { + if scope == "" { // outbound direct (possibly P2P) connection switch netutils.ClassifyIP(pkt.Info().Dst) { @@ -159,59 +165,82 @@ func GetConnection(id string) (*Connection, bool) { return conn, ok } -// Accept accepts the connection. -func (conn *Connection) Accept(reason string) { - if conn.SetVerdict(VerdictAccept) { - conn.Reason = reason +// AcceptWithContext accepts the connection. +func (conn *Connection) AcceptWithContext(reason string, ctx interface{}) { + if conn.SetVerdict(VerdictAccept, reason, ctx) { log.Infof("filter: granting connection %s, %s", conn, conn.Reason) } else { log.Warningf("filter: tried to accept %s, but current verdict is %s", conn, conn.Verdict) } } -// Block blocks the connection. -func (conn *Connection) Block(reason string) { - if conn.SetVerdict(VerdictBlock) { - conn.Reason = reason +// Accept is like AcceptWithContext but only accepts a reason. +func (conn *Connection) Accept(reason string) { + conn.AcceptWithContext(reason, nil) +} + +// BlockWithContext blocks the connection. +func (conn *Connection) BlockWithContext(reason string, ctx interface{}) { + if conn.SetVerdict(VerdictBlock, reason, ctx) { log.Infof("filter: blocking connection %s, %s", conn, conn.Reason) } else { log.Warningf("filter: tried to block %s, but current verdict is %s", conn, conn.Verdict) } } -// Drop drops the connection. -func (conn *Connection) Drop(reason string) { - if conn.SetVerdict(VerdictDrop) { - conn.Reason = reason +// Block is like BlockWithContext but does only accepts a reason. +func (conn *Connection) Block(reason string) { + conn.BlockWithContext(reason, nil) +} + +// DropWithContext drops the connection. +func (conn *Connection) DropWithContext(reason string, ctx interface{}) { + if conn.SetVerdict(VerdictDrop, reason, ctx) { log.Infof("filter: dropping connection %s, %s", conn, conn.Reason) } else { log.Warningf("filter: tried to drop %s, but current verdict is %s", conn, conn.Verdict) } } -// Deny blocks or drops the link depending on the connection direction. -func (conn *Connection) Deny(reason string) { +// Drop is like DropWithContext but does only accepts a reason. +func (conn *Connection) Drop(reason string) { + conn.DropWithContext(reason, nil) +} + +// DenyWithContext blocks or drops the link depending on the connection direction. +func (conn *Connection) DenyWithContext(reason string, ctx interface{}) { if conn.Inbound { - conn.Drop(reason) + conn.DropWithContext(reason, ctx) } else { - conn.Block(reason) + conn.BlockWithContext(reason, ctx) } } -// Failed marks the connection with VerdictFailed and stores the reason. -func (conn *Connection) Failed(reason string) { - if conn.SetVerdict(VerdictFailed) { - conn.Reason = reason +// Deny is like DenyWithContext but only accepts a reason. +func (conn *Connection) Deny(reason string) { + conn.DenyWithContext(reason, nil) +} + +// FailedWithContext marks the connection with VerdictFailed and stores the reason. +func (conn *Connection) FailedWithContext(reason string, ctx interface{}) { + if conn.SetVerdict(VerdictFailed, reason, ctx) { log.Infof("filter: dropping connection %s because of an internal error: %s", conn, reason) } else { log.Warningf("filter: tried to drop %s due to error but current verdict is %s", conn, conn.Verdict) } } +// Failed is like FailedWithContext but only accepts a string. +func (conn *Connection) Failed(reason string) { + conn.FailedWithContext(reason, nil) +} + // SetVerdict sets a new verdict for the connection, making sure it does not interfere with previous verdicts. -func (conn *Connection) SetVerdict(newVerdict Verdict) (ok bool) { +func (conn *Connection) SetVerdict(newVerdict Verdict, reason string, ctx interface{}) (ok bool) { if newVerdict >= conn.Verdict { conn.Verdict = newVerdict + conn.Reason = reason + conn.ReasonContext = ctx return true } return false @@ -229,39 +258,31 @@ func (conn *Connection) SaveWhenFinished() { // Save saves the connection in the storage and propagates the change through the database system. func (conn *Connection) Save() { - if conn.ID == "" { + conn.UpdateMeta() - // dns request - if !conn.KeyIsSet() { + if !conn.KeyIsSet() { + if conn.ID == "" { + // dns request + + // set key conn.SetKey(fmt.Sprintf("network:tree/%d/%s", conn.process.Pid, conn.Scope)) - conn.UpdateMeta() - } - // save to internal state - // check if it already exists - mapKey := strconv.Itoa(conn.process.Pid) + "/" + conn.Scope - dnsConnsLock.Lock() - _, ok := dnsConns[mapKey] - if !ok { + mapKey := strconv.Itoa(conn.process.Pid) + "/" + conn.Scope + + // save + dnsConnsLock.Lock() dnsConns[mapKey] = conn - } - dnsConnsLock.Unlock() + dnsConnsLock.Unlock() + } else { + // network connection - } else { - - // connection - if !conn.KeyIsSet() { + // set key conn.SetKey(fmt.Sprintf("network:tree/%d/%s/%s", conn.process.Pid, conn.Scope, conn.ID)) - conn.UpdateMeta() - } - // save to internal state - // check if it already exists - connsLock.Lock() - _, ok := conns[conn.ID] - if !ok { - conns[conn.ID] = conn - } - connsLock.Unlock() + // save + connsLock.Lock() + conns[conn.ID] = conn + connsLock.Unlock() + } } // notify database controller @@ -270,7 +291,11 @@ func (conn *Connection) Save() { // delete deletes a link from the storage and propagates the change. Nothing is locked - both the conns map and the connection itself require locking func (conn *Connection) delete() { - delete(conns, conn.ID) + if conn.ID == "" { + delete(dnsConns, strconv.Itoa(conn.process.Pid)+"/"+conn.Scope) + } else { + delete(conns, conn.ID) + } conn.Meta().Delete() dbController.PushUpdate(conn) diff --git a/network/database.go b/network/database.go index d7dca398..073dcbc0 100644 --- a/network/database.go +++ b/network/database.go @@ -77,7 +77,7 @@ func (s *StorageInterface) processQuery(q *query.Query, it *iterator.Iterator) { if slashes <= 1 { // processes for _, proc := range process.All() { - if strings.HasPrefix(proc.DatabaseKey(), q.DatabaseKeyPrefix()) { + if q.Matches(proc) { it.Next <- proc } } @@ -86,9 +86,9 @@ func (s *StorageInterface) processQuery(q *query.Query, it *iterator.Iterator) { if slashes <= 2 { // dns scopes only dnsConnsLock.RLock() - for _, dnsConns := range dnsConns { - if strings.HasPrefix(dnsConns.DatabaseKey(), q.DatabaseKeyPrefix()) { - it.Next <- dnsConns + for _, dnsConn := range dnsConns { + if q.Matches(dnsConn) { + it.Next <- dnsConn } } dnsConnsLock.RUnlock() @@ -98,7 +98,7 @@ func (s *StorageInterface) processQuery(q *query.Query, it *iterator.Iterator) { // connections connsLock.RLock() for _, conn := range conns { - if strings.HasPrefix(conn.DatabaseKey(), q.DatabaseKeyPrefix()) { + if q.Matches(conn) { it.Next <- conn } } diff --git a/network/dns.go b/network/dns.go index 88ca7be3..b33fa99f 100644 --- a/network/dns.go +++ b/network/dns.go @@ -5,6 +5,8 @@ import ( "strconv" "sync" "time" + + "github.com/safing/portmaster/process" ) var ( @@ -16,6 +18,9 @@ var ( // duration after which DNS requests without a following connection are logged openDNSRequestLimit = 3 * time.Second + + // scope prefix + unidentifiedProcessScopePrefix = strconv.Itoa(process.UnidentifiedProcessID) + "/" ) func removeOpenDNSRequest(pid int, fqdn string) { @@ -23,7 +28,13 @@ func removeOpenDNSRequest(pid int, fqdn string) { defer openDNSRequestsLock.Unlock() key := strconv.Itoa(pid) + "/" + fqdn - delete(openDNSRequests, key) + _, ok := openDNSRequests[key] + if ok { + delete(openDNSRequests, key) + } else if pid != process.UnidentifiedProcessID { + // check if there is an open dns request from an unidentified process + delete(openDNSRequests, unidentifiedProcessScopePrefix+fqdn) + } } // SaveOpenDNSRequest saves a dns request connection that was allowed to proceed. diff --git a/process/database.go b/process/database.go index b4ce09b3..1ce5295d 100644 --- a/process/database.go +++ b/process/database.go @@ -53,16 +53,13 @@ func (p *Process) Save() { p.Lock() defer p.Unlock() + p.UpdateMeta() + if !p.KeyIsSet() { + // set key p.SetKey(fmt.Sprintf("%s/%d", processDatabaseNamespace, p.Pid)) - p.CreateMeta() - } - processesLock.RLock() - _, ok := processes[p.Pid] - processesLock.RUnlock() - - if !ok { + // save processesLock.Lock() processes[p.Pid] = p processesLock.Unlock() @@ -113,7 +110,9 @@ func CleanProcessStorage(activePIDs map[int]struct{}) { _, active := activePIDs[p.Pid] switch { - case p.Pid <= 0: + case p.Pid == UnidentifiedProcessID: + // internal + case p.Pid == SystemProcessID: // internal case active: // process in system process table or recently seen on the network diff --git a/process/find.go b/process/find.go index 997d487b..30f93f2d 100644 --- a/process/find.go +++ b/process/find.go @@ -49,7 +49,7 @@ func GetPidByPacket(pkt packet.Packet) (pid int, direction bool, err error) { case pkt.Info().Protocol == packet.UDP && pkt.Info().Version == packet.IPv6: return getUDP6PacketInfo(localIP, localPort, remoteIP, remotePort, pkt.IsInbound()) default: - return -1, false, errors.New("unsupported protocol for finding process") + return UnidentifiedProcessID, false, errors.New("unsupported protocol for finding process") } } @@ -58,7 +58,7 @@ func GetPidByPacket(pkt packet.Packet) (pid int, direction bool, err error) { func GetProcessByPacket(pkt packet.Packet) (process *Process, direction bool, err error) { if !enableProcessDetection() { log.Tracer(pkt.Ctx()).Tracef("process: process detection disabled") - return UnknownProcess, direction, nil + return GetUnidentifiedProcess(pkt.Ctx()), pkt.Info().Direction, nil } log.Tracer(pkt.Ctx()).Tracef("process: getting process and profile by packet") @@ -107,7 +107,7 @@ func GetPidByEndpoints(localIP net.IP, localPort uint16, remoteIP net.IP, remote case protocol == packet.UDP && ipVersion == packet.IPv6: return getUDP6PacketInfo(localIP, localPort, remoteIP, remotePort, false) default: - return -1, false, errors.New("unsupported protocol for finding process") + return UnidentifiedProcessID, false, errors.New("unsupported protocol for finding process") } } @@ -116,7 +116,7 @@ func GetPidByEndpoints(localIP net.IP, localPort uint16, remoteIP net.IP, remote func GetProcessByEndpoints(ctx context.Context, localIP net.IP, localPort uint16, remoteIP net.IP, remotePort uint16, protocol packet.IPProtocol) (process *Process, err error) { if !enableProcessDetection() { log.Tracer(ctx).Tracef("process: process detection disabled") - return UnknownProcess, nil + return GetUnidentifiedProcess(ctx), nil } log.Tracer(ctx).Tracef("process: getting process and profile by endpoints") diff --git a/process/iphelper/get.go b/process/iphelper/get.go index 99c0f821..6487ea06 100644 --- a/process/iphelper/get.go +++ b/process/iphelper/get.go @@ -9,6 +9,10 @@ import ( "time" ) +const ( + unidentifiedProcessID = -1 +) + var ( tcp4Connections []*ConnectionEntry tcp4Listeners []*ConnectionEntry @@ -55,7 +59,7 @@ func GetTCP4PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote } lock.Unlock() if err != nil { - return -1, pktDirection, err + return unidentifiedProcessID, pktDirection, err } // search @@ -67,7 +71,7 @@ func GetTCP4PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote time.Sleep(waitTime) } - return -1, pktDirection, nil + return unidentifiedProcessID, pktDirection, nil } // GetTCP6PacketInfo returns the pid of the given IPv6/TCP connection. @@ -91,7 +95,7 @@ func GetTCP6PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote } lock.Unlock() if err != nil { - return -1, pktDirection, err + return unidentifiedProcessID, pktDirection, err } // search @@ -103,7 +107,7 @@ func GetTCP6PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote time.Sleep(waitTime) } - return -1, pktDirection, nil + return unidentifiedProcessID, pktDirection, nil } // GetUDP4PacketInfo returns the pid of the given IPv4/UDP connection. @@ -127,7 +131,7 @@ func GetUDP4PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote } lock.Unlock() if err != nil { - return -1, pktDirection, err + return unidentifiedProcessID, pktDirection, err } // search @@ -139,7 +143,7 @@ func GetUDP4PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote time.Sleep(waitTime) } - return -1, pktDirection, nil + return unidentifiedProcessID, pktDirection, nil } // GetUDP6PacketInfo returns the pid of the given IPv6/UDP connection. @@ -163,7 +167,7 @@ func GetUDP6PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote } lock.Unlock() if err != nil { - return -1, pktDirection, err + return unidentifiedProcessID, pktDirection, err } // search @@ -175,7 +179,7 @@ func GetUDP6PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remote time.Sleep(waitTime) } - return -1, pktDirection, nil + return unidentifiedProcessID, pktDirection, nil } func search(connections, listeners []*ConnectionEntry, localIP, remoteIP net.IP, localPort, remotePort uint16, pktDirection bool) (pid int, direction bool) { //nolint:unparam // TODO: use direction, it may not be used because results caused problems, investigate. @@ -204,7 +208,7 @@ func search(connections, listeners []*ConnectionEntry, localIP, remoteIP net.IP, } } - return -1, pktDirection + return unidentifiedProcessID, pktDirection } func searchConnections(list []*ConnectionEntry, localIP, remoteIP net.IP, localPort, remotePort uint16) (pid int) { @@ -218,7 +222,7 @@ func searchConnections(list []*ConnectionEntry, localIP, remoteIP net.IP, localP } } - return -1 + return unidentifiedProcessID } func searchListeners(list []*ConnectionEntry, localIP net.IP, localPort uint16) (pid int) { @@ -231,7 +235,7 @@ func searchListeners(list []*ConnectionEntry, localIP net.IP, localPort uint16) } } - return -1 + return unidentifiedProcessID } // GetActiveConnectionIDs returns all currently active connection IDs. diff --git a/process/proc/gather.go b/process/proc/gather.go index 436b0bb2..1413b3c9 100644 --- a/process/proc/gather.go +++ b/process/proc/gather.go @@ -33,7 +33,7 @@ func GetPidOfConnection(localIP net.IP, localPort uint16, protocol uint8) (pid i } } if !ok { - return -1, NoSocket + return unidentifiedProcessID, NoSocket } } @@ -45,7 +45,7 @@ func GetPidOfConnection(localIP net.IP, localPort uint16, protocol uint8) (pid i pid, ok = GetPidOfInode(uid, inode) } if !ok { - return -1, NoProcess + return unidentifiedProcessID, NoProcess } return @@ -64,7 +64,7 @@ func GetPidOfIncomingConnection(localIP net.IP, localPort uint16, protocol uint8 } if !ok { - return -1, NoSocket + return unidentifiedProcessID, NoSocket } } @@ -76,7 +76,7 @@ func GetPidOfIncomingConnection(localIP net.IP, localPort uint16, protocol uint8 pid, ok = GetPidOfInode(uid, inode) } if !ok { - return -1, NoProcess + return unidentifiedProcessID, NoProcess } return diff --git a/process/proc/get.go b/process/proc/get.go index dec27e23..52974b3e 100644 --- a/process/proc/get.go +++ b/process/proc/get.go @@ -7,6 +7,10 @@ import ( "net" ) +const ( + unidentifiedProcessID = -1 +) + // GetTCP4PacketInfo searches the network state tables for a TCP4 connection func GetTCP4PacketInfo(localIP net.IP, localPort uint16, remoteIP net.IP, remotePort uint16, pktDirection bool) (pid int, direction bool, err error) { return search(TCP4, localIP, localPort, pktDirection) @@ -52,11 +56,11 @@ func search(protocol uint8, localIP net.IP, localPort uint16, pktDirection bool) switch status { case NoSocket: - return -1, direction, errors.New("could not find socket") + return unidentifiedProcessID, direction, errors.New("could not find socket") case NoProcess: - return -1, direction, errors.New("could not find PID") + return unidentifiedProcessID, direction, errors.New("could not find PID") default: - return -1, direction, nil + return unidentifiedProcessID, direction, nil } } diff --git a/process/proc/processfinder.go b/process/proc/processfinder.go index 5ee1bb4b..5e6ed7cc 100644 --- a/process/proc/processfinder.go +++ b/process/proc/processfinder.go @@ -77,7 +77,7 @@ func GetPidOfInode(uid, inode int) (int, bool) { //nolint:gocognit // TODO } } - return -1, false + return unidentifiedProcessID, false } func findSocketFromPid(pid, inode int) bool { diff --git a/process/proc/sockets.go b/process/proc/sockets.go index c82b078c..bcdd91d4 100644 --- a/process/proc/sockets.go +++ b/process/proc/sockets.go @@ -100,7 +100,7 @@ func getConnectionSocket(localIP net.IP, localPort uint16, protocol uint8) (int, socketData, err := os.Open(procFile) if err != nil { log.Warningf("process/proc: could not read %s: %s", procFile, err) - return -1, -1, false + return unidentifiedProcessID, unidentifiedProcessID, false } defer socketData.Close() @@ -146,7 +146,7 @@ func getConnectionSocket(localIP net.IP, localPort uint16, protocol uint8) (int, } - return -1, -1, false + return unidentifiedProcessID, unidentifiedProcessID, false } @@ -187,7 +187,7 @@ func getListeningSocket(localIP net.IP, localPort uint16, protocol uint8) (uid, return data[0], data[1], true } - return -1, -1, false + return unidentifiedProcessID, unidentifiedProcessID, false } func procDelimiter(c rune) bool { diff --git a/process/process.go b/process/process.go index 2ab29383..8ef1ad73 100644 --- a/process/process.go +++ b/process/process.go @@ -75,11 +75,11 @@ func (p *Process) String() string { func GetOrFindPrimaryProcess(ctx context.Context, pid int) (*Process, error) { log.Tracer(ctx).Tracef("process: getting primary process for PID %d", pid) - if pid == -1 { - return UnknownProcess, nil - } - if pid == 0 { - return OSProcess, nil + switch pid { + case UnidentifiedProcessID: + return GetUnidentifiedProcess(ctx), nil + case SystemProcessID: + return GetSystemProcess(ctx), nil } process, err := loadProcess(ctx, pid) @@ -88,8 +88,8 @@ func GetOrFindPrimaryProcess(ctx context.Context, pid int) (*Process, error) { } for { - if process.ParentPid == 0 { - return OSProcess, nil + if process.ParentPid <= 0 { + return process, nil } parentProcess, err := loadProcess(ctx, process.ParentPid) if err != nil { @@ -121,11 +121,11 @@ func GetOrFindPrimaryProcess(ctx context.Context, pid int) (*Process, error) { func GetOrFindProcess(ctx context.Context, pid int) (*Process, error) { log.Tracer(ctx).Tracef("process: getting process for PID %d", pid) - if pid == -1 { - return UnknownProcess, nil - } - if pid == 0 { - return OSProcess, nil + switch pid { + case UnidentifiedProcessID: + return GetUnidentifiedProcess(ctx), nil + case SystemProcessID: + return GetSystemProcess(ctx), nil } p, err := loadProcess(ctx, pid) @@ -184,11 +184,12 @@ func deduplicateRequest(ctx context.Context, pid int) (finishRequest func()) { } func loadProcess(ctx context.Context, pid int) (*Process, error) { - if pid == -1 { - return UnknownProcess, nil - } - if pid == 0 { - return OSProcess, nil + + switch pid { + case UnidentifiedProcessID: + return GetUnidentifiedProcess(ctx), nil + case SystemProcessID: + return GetSystemProcess(ctx), nil } process, ok := GetProcessFromStorage(pid) diff --git a/process/profile.go b/process/profile.go index 388684fa..0f0ad5c6 100644 --- a/process/profile.go +++ b/process/profile.go @@ -15,6 +15,8 @@ func (p *Process) GetProfile(ctx context.Context) error { // only find profiles if not already done. if p.profile != nil { log.Tracer(ctx).Trace("process: profile already loaded") + // mark profile as used + p.profile.MarkUsed() return nil } log.Tracer(ctx).Trace("process: loading profile") @@ -29,10 +31,8 @@ func (p *Process) GetProfile(ctx context.Context) error { localProfile.Name = p.ExecName } - // mark as used and save - if localProfile.MarkUsed() { - _ = localProfile.Save() - } + // mark profile as used + localProfile.MarkUsed() p.LocalProfileKey = localProfile.Key() p.profile = profile.NewLayeredProfile(localProfile) diff --git a/process/special.go b/process/special.go new file mode 100644 index 00000000..277d337b --- /dev/null +++ b/process/special.go @@ -0,0 +1,84 @@ +package process + +import ( + "context" + "time" + + "github.com/safing/portbase/log" + "github.com/safing/portmaster/profile" +) + +// Special Process IDs +const ( + UnidentifiedProcessID = -1 + SystemProcessID = 0 +) + +var ( + // unidentifiedProcess is used when a process cannot be found. + unidentifiedProcess = &Process{ + UserID: UnidentifiedProcessID, + UserName: "Unknown", + Pid: UnidentifiedProcessID, + ParentPid: UnidentifiedProcessID, + Name: "Unidentified Processes", + } + + // systemProcess is used to represent the Kernel. + systemProcess = &Process{ + UserID: SystemProcessID, + UserName: "Kernel", + Pid: SystemProcessID, + ParentPid: SystemProcessID, + Name: "Operating System", + } +) + +// GetUnidentifiedProcess returns the special process assigned to unidentified processes. +func GetUnidentifiedProcess(ctx context.Context) *Process { + return getSpecialProcess(ctx, UnidentifiedProcessID, unidentifiedProcess, profile.GetUnidentifiedProfile) +} + +// GetSystemProcess returns the special process used for the Kernel. +func GetSystemProcess(ctx context.Context) *Process { + return getSpecialProcess(ctx, SystemProcessID, systemProcess, profile.GetSystemProfile) +} + +func getSpecialProcess(ctx context.Context, pid int, template *Process, getProfile func() *profile.Profile) *Process { + // check storage + p, ok := GetProcessFromStorage(pid) + if ok { + return p + } + + // assign template + p = template + + p.Lock() + defer p.Unlock() + + if p.FirstSeen == 0 { + p.FirstSeen = time.Now().Unix() + } + + // only find profiles if not already done. + if p.profile != nil { + log.Tracer(ctx).Trace("process: special profile already loaded") + // mark profile as used + p.profile.MarkUsed() + return p + } + log.Tracer(ctx).Trace("process: loading special profile") + + // get profile + localProfile := getProfile() + + // mark profile as used + localProfile.MarkUsed() + + p.LocalProfileKey = localProfile.Key() + p.profile = profile.NewLayeredProfile(localProfile) + + go p.Save() + return p +} diff --git a/process/unknown.go b/process/unknown.go deleted file mode 100644 index 4f6cde12..00000000 --- a/process/unknown.go +++ /dev/null @@ -1,26 +0,0 @@ -package process - -var ( - // UnknownProcess is used when a process cannot be found. - UnknownProcess = &Process{ - UserID: -1, - UserName: "Unknown", - Pid: -1, - ParentPid: -1, - Name: "Unknown Processes", - } - - // OSProcess is used to represent the Kernel. - OSProcess = &Process{ - UserID: 0, - UserName: "Kernel", - Pid: 0, - ParentPid: 0, - Name: "Operating System", - } -) - -func init() { - UnknownProcess.Save() - OSProcess.Save() -} diff --git a/profile/active.go b/profile/active.go index ff8f3d3e..ff6a71c8 100644 --- a/profile/active.go +++ b/profile/active.go @@ -1,7 +1,14 @@ package profile import ( + "context" "sync" + "time" +) + +const ( + activeProfileCleanerTickDuration = 10 * time.Minute + activeProfileCleanerThreshold = 1 * time.Hour ) var ( @@ -38,7 +45,34 @@ func markActiveProfileAsOutdated(scopedID string) { profile, ok := activeProfiles[scopedID] if ok { - profile.oudated.Set() + profile.outdated.Set() delete(activeProfiles, scopedID) } } + +func cleanActiveProfiles(ctx context.Context) error { + for { + select { + case <-time.After(activeProfileCleanerTickDuration): + + threshold := time.Now().Add(-activeProfileCleanerThreshold) + + activeProfilesLock.Lock() + for id, profile := range activeProfiles { + // get last used + profile.Lock() + lastUsed := profile.lastUsed + profile.Unlock() + // remove if not used for a while + if lastUsed.Before(threshold) { + profile.outdated.Set() + delete(activeProfiles, id) + } + } + activeProfilesLock.Unlock() + + case <-ctx.Done(): + return nil + } + } +} diff --git a/profile/config-update.go b/profile/config-update.go index 15e62b8e..b4c3f5e4 100644 --- a/profile/config-update.go +++ b/profile/config-update.go @@ -70,8 +70,8 @@ func updateGlobalConfigProfile(ctx context.Context, data interface{}) error { // build global profile for reference profile := &Profile{ - ID: "config", - Source: SourceGlobal, + ID: "global-config", + Source: SourceSpecial, Name: "Global Configuration", Config: make(map[string]interface{}), internalSave: true, diff --git a/profile/config.go b/profile/config.go index 9cc341f9..8842530e 100644 --- a/profile/config.go +++ b/profile/config.go @@ -30,6 +30,9 @@ var ( CfgOptionFilterSubDomainsKey = "filter/includeSubdomains" cfgOptionFilterSubDomains config.IntOption // security level option + CfgOptionFilterCNAMEKey = "filter/includeCNAMEs" + cfgOptionFilterCNAME config.IntOption // security level option + CfgOptionBlockScopeLocalKey = "filter/blockLocal" cfgOptionBlockScopeLocal config.IntOption // security level option @@ -53,6 +56,9 @@ var ( CfgOptionRemoveBlockedDNSKey = "filter/removeBlockedDNS" cfgOptionRemoveBlockedDNS config.IntOption // security level option + + CfgOptionPreventBypassingKey = "filter/preventBypassing" + cfgOptionPreventBypassing config.IntOption // security level option ) func registerConfiguration() error { @@ -177,6 +183,24 @@ Examples: cfgOptionFilterLists = config.Concurrent.GetAsStringArray(CfgOptionFilterListKey, []string{}) cfgStringArrayOptions[CfgOptionFilterListKey] = cfgOptionFilterLists + // Include CNAMEs + err = config.Register(&config.Option{ + Name: "Filter CNAMEs", + Key: CfgOptionFilterCNAMEKey, + Description: "Also filter requests where a CNAME would be blocked", + OptType: config.OptTypeInt, + ExternalOptType: "security level", + DefaultValue: status.SecurityLevelsAll, + ValidationRegex: "^(7|6|4)$", + ExpertiseLevel: config.ExpertiseLevelExpert, + }) + if err != nil { + return err + } + cfgOptionFilterCNAME = config.Concurrent.GetAsInt(CfgOptionFilterCNAMEKey, int64(status.SecurityLevelsAll)) + cfgIntOptions[CfgOptionFilterCNAMEKey] = cfgOptionFilterCNAME + + // Include subdomains err = config.Register(&config.Option{ Name: "Filter SubDomains", Key: CfgOptionFilterSubDomainsKey, @@ -325,5 +349,22 @@ Examples: cfgOptionRemoveBlockedDNS = config.Concurrent.GetAsInt(CfgOptionRemoveBlockedDNSKey, int64(status.SecurityLevelsAll)) cfgIntOptions[CfgOptionRemoveBlockedDNSKey] = cfgOptionRemoveBlockedDNS + err = config.Register(&config.Option{ + Name: "Prevent Bypassing", + Key: CfgOptionPreventBypassingKey, + Description: "Prevent apps from bypassing the privacy filter: Firefox by disabling DNS-over-HTTPs", + OptType: config.OptTypeInt, + ExpertiseLevel: config.ExpertiseLevelUser, + ReleaseLevel: config.ReleaseLevelBeta, + ExternalOptType: "security level", + DefaultValue: status.SecurityLevelsAll, + ValidationRegex: "^(7|6|4)", + }) + if err != nil { + return err + } + cfgOptionPreventBypassing = config.Concurrent.GetAsInt((CfgOptionPreventBypassingKey), int64(status.SecurityLevelsAll)) + cfgIntOptions[CfgOptionPreventBypassingKey] = cfgOptionPreventBypassing + return nil } diff --git a/profile/endpoints/endpoint-any.go b/profile/endpoints/endpoint-any.go index 34fd65b4..8e8deb98 100644 --- a/profile/endpoints/endpoint-any.go +++ b/profile/endpoints/endpoint-any.go @@ -8,8 +8,8 @@ type EndpointAny struct { } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointAny) Matches(entity *intel.Entity) (result EPResult, reason string) { - return ep.matchesPPP(entity), "matches *" +func (ep *EndpointAny) Matches(entity *intel.Entity) (EPResult, Reason) { + return ep.match(ep, entity, "*", "matches") } func (ep *EndpointAny) String() string { diff --git a/profile/endpoints/endpoint-asn.go b/profile/endpoints/endpoint-asn.go index dee94f2d..6713d199 100644 --- a/profile/endpoints/endpoint-asn.go +++ b/profile/endpoints/endpoint-asn.go @@ -16,24 +16,22 @@ var ( type EndpointASN struct { EndpointBase - ASN uint - Reason string + ASN uint } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointASN) Matches(entity *intel.Entity) (result EPResult, reason string) { - if entity.IP == nil { - return Undeterminable, "" - } - +func (ep *EndpointASN) Matches(entity *intel.Entity) (EPResult, Reason) { asn, ok := entity.GetASN() if !ok { - return Undeterminable, "" + return Undeterminable, nil } + if asn == ep.ASN { - return ep.matchesPPP(entity), ep.Reason + asnStr := strconv.Itoa(int(ep.ASN)) + return ep.match(ep, entity, asnStr, "IP is part of AS") } - return NoMatch, "" + + return NoMatch, nil } func (ep *EndpointASN) String() string { @@ -48,8 +46,7 @@ func parseTypeASN(fields []string) (Endpoint, error) { } ep := &EndpointASN{ - ASN: uint(asn), - Reason: "IP is part of AS" + strconv.FormatInt(int64(asn), 10), + ASN: uint(asn), } return ep.parsePPP(ep, fields) } diff --git a/profile/endpoints/endpoint-country.go b/profile/endpoints/endpoint-country.go index 63d21bfc..85449cf5 100644 --- a/profile/endpoints/endpoint-country.go +++ b/profile/endpoints/endpoint-country.go @@ -19,19 +19,16 @@ type EndpointCountry struct { } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointCountry) Matches(entity *intel.Entity) (result EPResult, reason string) { - if entity.IP == nil { - return Undeterminable, "" - } - +func (ep *EndpointCountry) Matches(entity *intel.Entity) (EPResult, Reason) { country, ok := entity.GetCountry() if !ok { - return Undeterminable, "" + return Undeterminable, nil } + if country == ep.Country { - return ep.matchesPPP(entity), "IP is located in " + country + return ep.match(ep, entity, country, "IP is located in") } - return NoMatch, "" + return NoMatch, nil } func (ep *EndpointCountry) String() string { diff --git a/profile/endpoints/endpoint-domain.go b/profile/endpoints/endpoint-domain.go index 225a4b5d..fbd0dcf9 100644 --- a/profile/endpoints/endpoint-domain.go +++ b/profile/endpoints/endpoint-domain.go @@ -28,42 +28,60 @@ type EndpointDomain struct { Domain string DomainZone string MatchType uint8 - Reason string } -// Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointDomain) Matches(entity *intel.Entity) (result EPResult, reason string) { - if entity.Domain == "" { - return NoMatch, "" - } +func (ep *EndpointDomain) check(entity *intel.Entity, domain string) (EPResult, Reason) { + result, reason := ep.match(ep, entity, ep.Domain, "domain matches") switch ep.MatchType { case domainMatchTypeExact: - if entity.Domain == ep.Domain { - return ep.matchesPPP(entity), ep.Reason + if domain == ep.Domain { + return result, reason } case domainMatchTypeZone: - if entity.Domain == ep.Domain { - return ep.matchesPPP(entity), ep.Reason + if domain == ep.Domain { + return result, reason } - if strings.HasSuffix(entity.Domain, ep.DomainZone) { - return ep.matchesPPP(entity), ep.Reason + if strings.HasSuffix(domain, ep.DomainZone) { + return result, reason } case domainMatchTypeSuffix: - if strings.HasSuffix(entity.Domain, ep.Domain) { - return ep.matchesPPP(entity), ep.Reason + if strings.HasSuffix(domain, ep.Domain) { + return result, reason } case domainMatchTypePrefix: - if strings.HasPrefix(entity.Domain, ep.Domain) { - return ep.matchesPPP(entity), ep.Reason + if strings.HasPrefix(domain, ep.Domain) { + return result, reason } case domainMatchTypeContains: - if strings.Contains(entity.Domain, ep.Domain) { - return ep.matchesPPP(entity), ep.Reason + if strings.Contains(domain, ep.Domain) { + return result, reason + } + } + return NoMatch, nil +} + +// Matches checks whether the given entity matches this endpoint definition. +func (ep *EndpointDomain) Matches(entity *intel.Entity) (EPResult, Reason) { + if entity.Domain == "" { + return NoMatch, nil + } + + result, reason := ep.check(entity, entity.Domain) + if result != NoMatch { + return result, reason + } + + if entity.CNAMECheckEnabled() { + for _, domain := range entity.CNAME { + result, reason = ep.check(entity, domain) + if result == Denied { + return result, reason + } } } - return NoMatch, "" + return NoMatch, nil } func (ep *EndpointDomain) String() string { @@ -76,7 +94,6 @@ func parseTypeDomain(fields []string) (Endpoint, error) { if domainRegex.MatchString(domain) || altDomainRegex.MatchString(domain) { ep := &EndpointDomain{ OriginalValue: domain, - Reason: "domain matches " + domain, } // fix domain ending diff --git a/profile/endpoints/endpoint-ip.go b/profile/endpoints/endpoint-ip.go index 6fa742c0..43ea47f7 100644 --- a/profile/endpoints/endpoint-ip.go +++ b/profile/endpoints/endpoint-ip.go @@ -10,19 +10,19 @@ import ( type EndpointIP struct { EndpointBase - IP net.IP - Reason string + IP net.IP } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointIP) Matches(entity *intel.Entity) (result EPResult, reason string) { +func (ep *EndpointIP) Matches(entity *intel.Entity) (EPResult, Reason) { if entity.IP == nil { - return Undeterminable, "" + return Undeterminable, nil } + if ep.IP.Equal(entity.IP) { - return ep.matchesPPP(entity), ep.Reason + return ep.match(ep, entity, ep.IP.String(), "IP matches") } - return NoMatch, "" + return NoMatch, nil } func (ep *EndpointIP) String() string { @@ -33,8 +33,7 @@ func parseTypeIP(fields []string) (Endpoint, error) { ip := net.ParseIP(fields[1]) if ip != nil { ep := &EndpointIP{ - IP: ip, - Reason: "IP is " + ip.String(), + IP: ip, } return ep.parsePPP(ep, fields) } diff --git a/profile/endpoints/endpoint-iprange.go b/profile/endpoints/endpoint-iprange.go index 16afa4bb..bc0d22fe 100644 --- a/profile/endpoints/endpoint-iprange.go +++ b/profile/endpoints/endpoint-iprange.go @@ -10,19 +10,18 @@ import ( type EndpointIPRange struct { EndpointBase - Net *net.IPNet - Reason string + Net *net.IPNet } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointIPRange) Matches(entity *intel.Entity) (result EPResult, reason string) { +func (ep *EndpointIPRange) Matches(entity *intel.Entity) (EPResult, Reason) { if entity.IP == nil { - return Undeterminable, "" + return Undeterminable, nil } if ep.Net.Contains(entity.IP) { - return ep.matchesPPP(entity), ep.Reason + return ep.match(ep, entity, ep.Net.String(), "IP is in") } - return NoMatch, "" + return NoMatch, nil } func (ep *EndpointIPRange) String() string { @@ -33,8 +32,7 @@ func parseTypeIPRange(fields []string) (Endpoint, error) { _, net, err := net.ParseCIDR(fields[1]) if err == nil { ep := &EndpointIPRange{ - Net: net, - Reason: "IP is part of " + net.String(), + Net: net, } return ep.parsePPP(ep, fields) } diff --git a/profile/endpoints/endpoint-lists.go b/profile/endpoints/endpoint-lists.go index fe151347..27ec8b00 100644 --- a/profile/endpoints/endpoint-lists.go +++ b/profile/endpoints/endpoint-lists.go @@ -10,22 +10,21 @@ import ( type EndpointLists struct { EndpointBase - ListSet *intel.ListSet + ListSet []string Lists string - Reason string } // Matches checks whether the given entity matches this endpoint definition. -func (ep *EndpointLists) Matches(entity *intel.Entity) (result EPResult, reason string) { - lists, ok := entity.GetLists() - if !ok { - return Undeterminable, "" +func (ep *EndpointLists) Matches(entity *intel.Entity) (EPResult, Reason) { + if !entity.LoadLists() { + return Undeterminable, nil } - matched := ep.ListSet.MatchSet(lists) - if len(matched) > 0 { - return ep.matchesPPP(entity), ep.Reason + + if entity.MatchLists(ep.ListSet) { + return ep.match(ep, entity, ep.Lists, "filterlist contains", "filterlist", entity.ListBlockReason()) } - return NoMatch, "" + + return NoMatch, nil } func (ep *EndpointLists) String() string { @@ -36,9 +35,8 @@ func parseTypeList(fields []string) (Endpoint, error) { if strings.HasPrefix(fields[1], "L:") { lists := strings.Split(strings.TrimPrefix(fields[1], "L:"), ",") ep := &EndpointLists{ - ListSet: intel.NewListSet(lists), + ListSet: lists, Lists: "L:" + strings.Join(lists, ","), - Reason: "matched lists " + strings.Join(lists, ","), } return ep.parsePPP(ep, fields) } diff --git a/profile/endpoints/endpoint.go b/profile/endpoints/endpoint.go index 687834cd..76847ac7 100644 --- a/profile/endpoints/endpoint.go +++ b/profile/endpoints/endpoint.go @@ -11,7 +11,7 @@ import ( // Endpoint describes an Endpoint Matcher type Endpoint interface { - Matches(entity *intel.Entity) (result EPResult, reason string) + Matches(entity *intel.Entity) (EPResult, Reason) String() string } @@ -24,6 +24,35 @@ type EndpointBase struct { //nolint:maligned // TODO Permitted bool } +func (ep *EndpointBase) match(s fmt.Stringer, entity *intel.Entity, value, desc string, keyval ...interface{}) (EPResult, Reason) { + result := ep.matchesPPP(entity) + if result == Undeterminable || result == NoMatch { + return result, nil + } + + return result, ep.makeReason(s, value, desc, keyval...) +} + +func (ep *EndpointBase) makeReason(s fmt.Stringer, value, desc string, keyval ...interface{}) Reason { + r := &reason{ + description: desc, + Filter: ep.renderPPP(s.String()), + Permitted: ep.Permitted, + Value: value, + } + + r.Extra = make(map[string]interface{}) + + for idx := 0; idx < len(keyval)/2; idx += 2 { + key := keyval[idx] + val := keyval[idx+1] + + r.Extra[key.(string)] = val + } + + return r +} + func (ep *EndpointBase) matchesPPP(entity *intel.Entity) (result EPResult) { // only check if protocol is defined if ep.Protocol > 0 { diff --git a/profile/endpoints/endpoints.go b/profile/endpoints/endpoints.go index 7a49a0c8..f74edd24 100644 --- a/profile/endpoints/endpoints.go +++ b/profile/endpoints/endpoints.go @@ -21,6 +21,12 @@ const ( Permitted ) +// IsDecision returns true if result represents a decision +// and false if result is NoMatch or Undeterminable. +func IsDecision(result EPResult) bool { + return result == Denied || result == Permitted || result == Undeterminable +} + // ParseEndpoints parses a list of endpoints and returns a list of Endpoints for matching. func ParseEndpoints(entries []string) (Endpoints, error) { var firstErr error @@ -57,7 +63,7 @@ func (e Endpoints) IsSet() bool { } // Match checks whether the given entity matches any of the endpoint definitions in the list. -func (e Endpoints) Match(entity *intel.Entity) (result EPResult, reason string) { +func (e Endpoints) Match(entity *intel.Entity) (result EPResult, reason Reason) { for _, entry := range e { if entry != nil { if result, reason = entry.Matches(entity); result != NoMatch { @@ -66,7 +72,7 @@ func (e Endpoints) Match(entity *intel.Entity) (result EPResult, reason string) } } - return NoMatch, "" + return NoMatch, nil } func (e Endpoints) String() string { diff --git a/profile/endpoints/reason.go b/profile/endpoints/reason.go new file mode 100644 index 00000000..d137c3b3 --- /dev/null +++ b/profile/endpoints/reason.go @@ -0,0 +1,34 @@ +package endpoints + +// Reason describes the reason why an endpoint has been +// permitted or blocked. +type Reason interface { + // String should return a human readable string + // describing the decision reason. + String() string + + // Context returns the context that was used + // for the decision. + Context() interface{} +} + +type reason struct { + description string + Filter string + Value string + Permitted bool + Extra map[string]interface{} +} + +func (r *reason) String() string { + prefix := "endpoint in blocklist: " + if r.Permitted { + prefix = "endpoint in whitelist: " + } + + return prefix + r.description + " " + r.Value +} + +func (r *reason) Context() interface{} { + return r +} diff --git a/profile/module.go b/profile/module.go index 615eddcf..8d962a8a 100644 --- a/profile/module.go +++ b/profile/module.go @@ -42,6 +42,8 @@ func start() error { return err } + module.StartServiceWorker("clean active profiles", 0, cleanActiveProfiles) + err = updateGlobalConfigProfile(module.Ctx, nil) if err != nil { log.Warningf("profile: error during loading global profile from configuration: %s", err) diff --git a/profile/profile-layered.go b/profile/profile-layered.go index 90f0478b..45311662 100644 --- a/profile/profile-layered.go +++ b/profile/profile-layered.go @@ -43,6 +43,8 @@ type LayeredProfile struct { RemoveOutOfScopeDNS config.BoolOption RemoveBlockedDNS config.BoolOption FilterSubDomains config.BoolOption + FilterCNAMEs config.BoolOption + PreventBypassing config.BoolOption } // NewLayeredProfile returns a new layered profile based on the given local profile. @@ -98,6 +100,14 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile { CfgOptionFilterSubDomainsKey, cfgOptionFilterSubDomains, ) + new.FilterCNAMEs = new.wrapSecurityLevelOption( + CfgOptionFilterCNAMEKey, + cfgOptionFilterCNAME, + ) + new.PreventBypassing = new.wrapSecurityLevelOption( + CfgOptionPreventBypassingKey, + cfgOptionPreventBypassing, + ) // TODO: load linked profiles. @@ -123,7 +133,7 @@ func (lp *LayeredProfile) Update() (revisionCounter uint64) { var changed bool for i, layer := range lp.layers { - if layer.oudated.IsSet() { + if layer.outdated.IsSet() { changed = true // update layer newLayer, err := GetProfile(layer.Source, layer.ID) @@ -170,6 +180,11 @@ func (lp *LayeredProfile) updateCaches() { // TODO: ignore community profiles } +// MarkUsed marks the localProfile as used. +func (lp *LayeredProfile) MarkUsed() { + lp.localProfile.MarkUsed() +} + // SecurityLevel returns the highest security level of all layered profiles. func (lp *LayeredProfile) SecurityLevel() uint8 { return uint8(atomic.LoadUint32(lp.securityLevel)) @@ -189,12 +204,12 @@ func (lp *LayeredProfile) DefaultAction() uint8 { } // MatchEndpoint checks if the given endpoint matches an entry in any of the profiles. -func (lp *LayeredProfile) MatchEndpoint(entity *intel.Entity) (result endpoints.EPResult, reason string) { +func (lp *LayeredProfile) MatchEndpoint(entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) { for _, layer := range lp.layers { if layer.endpoints.IsSet() { - result, reason = layer.endpoints.Match(entity) - if result != endpoints.NoMatch { - return + result, reason := layer.endpoints.Match(entity) + if endpoints.IsDecision(result) { + return result, reason } } } @@ -205,14 +220,14 @@ func (lp *LayeredProfile) MatchEndpoint(entity *intel.Entity) (result endpoints. } // MatchServiceEndpoint checks if the given endpoint of an inbound connection matches an entry in any of the profiles. -func (lp *LayeredProfile) MatchServiceEndpoint(entity *intel.Entity) (result endpoints.EPResult, reason string) { +func (lp *LayeredProfile) MatchServiceEndpoint(entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) { entity.EnableReverseResolving() for _, layer := range lp.layers { if layer.serviceEndpoints.IsSet() { - result, reason = layer.serviceEndpoints.Match(entity) - if result != endpoints.NoMatch { - return + result, reason := layer.serviceEndpoints.Match(entity) + if endpoints.IsDecision(result) { + return result, reason } } } @@ -224,33 +239,34 @@ func (lp *LayeredProfile) MatchServiceEndpoint(entity *intel.Entity) (result end // MatchFilterLists matches the entity against the set of filter // lists. -func (lp *LayeredProfile) MatchFilterLists(entity *intel.Entity) (result endpoints.EPResult, reason string) { +func (lp *LayeredProfile) MatchFilterLists(entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) { entity.ResolveSubDomainLists(lp.FilterSubDomains()) - - lookupMap, hasLists := entity.GetListsMap() - if !hasLists { - return endpoints.NoMatch, "" - } + entity.EnableCNAMECheck(lp.FilterCNAMEs()) for _, layer := range lp.layers { - if reason := lookupMap.Match(layer.filterListIDs); reason != "" { - return endpoints.Denied, reason - } - - // only check the first layer that has filter list - // IDs defined. + // search for the first layer that has filterListIDs set if len(layer.filterListIDs) > 0 { - return endpoints.NoMatch, "" + entity.LoadLists() + + if entity.MatchLists(layer.filterListIDs) { + return endpoints.Denied, entity.ListBlockReason() + } + + return endpoints.NoMatch, nil } } cfgLock.RLock() defer cfgLock.RUnlock() - if reason := lookupMap.Match(cfgFilterLists); reason != "" { - return endpoints.Denied, reason + if len(cfgFilterLists) > 0 { + entity.LoadLists() + + if entity.MatchLists(cfgFilterLists) { + return endpoints.Denied, entity.ListBlockReason() + } } - return endpoints.NoMatch, "" + return endpoints.NoMatch, nil } // AddEndpoint adds an endpoint to the local endpoint list, saves the local profile and reloads the configuration. diff --git a/profile/profile.go b/profile/profile.go index 46e3b447..70f7c162 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -19,15 +19,15 @@ import ( ) var ( - lastUsedUpdateThreshold = 1 * time.Hour + lastUsedUpdateThreshold = 24 * time.Hour ) // Profile Sources const ( - SourceLocal string = "local" + SourceLocal string = "local" // local, editable + SourceSpecial string = "special" // specials (read-only) SourceCommunity string = "community" SourceEnterprise string = "enterprise" - SourceGlobal string = "global" ) // Default Action IDs @@ -77,7 +77,8 @@ type Profile struct { //nolint:maligned // not worth the effort filterListIDs []string // Lifecycle Management - oudated *abool.AtomicBool + outdated *abool.AtomicBool + lastUsed time.Time // Framework // If a Profile is declared as a Framework (i.e. an Interpreter and the likes), then the real process/actor must be found @@ -94,7 +95,7 @@ type Profile struct { //nolint:maligned // not worth the effort func (profile *Profile) prepConfig() (err error) { // prepare configuration profile.configPerspective, err = config.NewPerspective(profile.Config) - profile.oudated = abool.New() + profile.outdated = abool.New() return } @@ -156,10 +157,11 @@ func (profile *Profile) parseConfig() error { // New returns a new Profile. func New() *Profile { profile := &Profile{ - ID: uuid.NewV4().String(), - Source: SourceLocal, - Created: time.Now().Unix(), - Config: make(map[string]interface{}), + ID: uuid.NewV4().String(), + Source: SourceLocal, + Created: time.Now().Unix(), + Config: make(map[string]interface{}), + internalSave: true, } // create placeholders @@ -190,13 +192,26 @@ func (profile *Profile) Save() error { return profileDB.Put(profile) } -// MarkUsed marks the profile as used, eventually. -func (profile *Profile) MarkUsed() (updated bool) { +// MarkUsed marks the profile as used and saves it when it has changed. +func (profile *Profile) MarkUsed() { + profile.Lock() + // lastUsed + profile.lastUsed = time.Now() + + // ApproxLastUsed + save := false if time.Now().Add(-lastUsedUpdateThreshold).Unix() > profile.ApproxLastUsed { profile.ApproxLastUsed = time.Now().Unix() - return true + save = true + } + profile.Unlock() + + if save { + err := profile.Save() + if err != nil { + log.Warningf("profiles: failed to save profile %s after marking as used: %s", profile.ScopedID(), err) + } } - return false } // String returns a string representation of the Profile. @@ -224,8 +239,6 @@ func (profile *Profile) addEndpointyEntry(cfgKey, newEntry string) { endpointList = append(endpointList, newEntry) profile.Config[cfgKey] = endpointList - // save without full reload - profile.internalSave = true profile.Unlock() err := profile.Save() if err != nil { @@ -233,10 +246,13 @@ func (profile *Profile) addEndpointyEntry(cfgKey, newEntry string) { } // reload manually + profile.Lock() + profile.dataParsed = false err = profile.parseConfig() if err != nil { log.Warningf("profile: failed to parse profile config after adding endpoint: %s", err) } + profile.Unlock() } // GetProfile loads a profile from the database. @@ -249,6 +265,7 @@ func GetProfileByScopedID(scopedID string) (*Profile, error) { // check cache profile := getActiveProfile(scopedID) if profile != nil { + profile.MarkUsed() return profile, nil } @@ -266,7 +283,6 @@ func GetProfileByScopedID(scopedID string) (*Profile, error) { // lock for prepping profile.Lock() - defer profile.Unlock() // prepare config err = profile.prepConfig() @@ -280,7 +296,13 @@ func GetProfileByScopedID(scopedID string) (*Profile, error) { log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err) } + // mark as internal + profile.internalSave = true + + profile.Unlock() + // mark active + profile.MarkUsed() markProfileActive(profile) return profile, nil diff --git a/profile/special.go b/profile/special.go new file mode 100644 index 00000000..6bce01d7 --- /dev/null +++ b/profile/special.go @@ -0,0 +1,56 @@ +package profile + +import ( + "github.com/safing/portbase/log" +) + +const ( + unidentifiedProfileID = "_unidentified" + systemProfileID = "_system" +) + +// GetUnidentifiedProfile returns the special profile assigned to unidentified processes. +func GetUnidentifiedProfile() *Profile { + // get profile + profile, err := GetProfile(SourceLocal, unidentifiedProfileID) + if err == nil { + return profile + } + + // create if not available (or error) + profile = New() + profile.Name = "Unidentified Processes" + profile.Source = SourceLocal + profile.ID = unidentifiedProfileID + + // save to db + err = profile.Save() + if err != nil { + log.Warningf("profiles: failed to save %s: %s", profile.ScopedID(), err) + } + + return profile +} + +// GetSystemProfile returns the special profile used for the Kernel. +func GetSystemProfile() *Profile { + // get profile + profile, err := GetProfile(SourceLocal, systemProfileID) + if err == nil { + return profile + } + + // create if not available (or error) + profile = New() + profile.Name = "Operating System" + profile.Source = SourceLocal + profile.ID = systemProfileID + + // save to db + err = profile.Save() + if err != nil { + log.Warningf("profiles: failed to save %s: %s", profile.ScopedID(), err) + } + + return profile +} diff --git a/resolver/block_detection.go b/resolver/block_detection.go new file mode 100644 index 00000000..8a4005cd --- /dev/null +++ b/resolver/block_detection.go @@ -0,0 +1,61 @@ +package resolver + +import ( + "net" + + "github.com/miekg/dns" +) + +// Supported upstream block detections +const ( + BlockDetectionRefused = "refused" + BlockDetectionZeroIP = "zeroip" + BlockDetectionEmptyAnswer = "empty" + BlockDetectionDisabled = "disabled" +) + +func isBlockedUpstream(resolver *Resolver, answer *dns.Msg) bool { + if resolver.UpstreamBlockDetection == BlockDetectionDisabled { + return false + } + + switch resolver.UpstreamBlockDetection { + case BlockDetectionRefused: + return answer.Rcode == dns.RcodeRefused + case BlockDetectionZeroIP: + if answer.Rcode != dns.RcodeSuccess { + return false + } + var ips []net.IP + for _, rr := range answer.Answer { + switch v := rr.(type) { + case *dns.A: + ips = append(ips, v.A) + case *dns.AAAA: + ips = append(ips, v.AAAA) + } + } + + if len(ips) == 0 { + return false // we expected an empty IP + } + + for _, ip := range ips { + if ip.To4() != nil { + if !ip.Equal(net.IPv4zero) { + return false + } + } else { + if !ip.To16().Equal(net.IPv6zero) { + return false + } + } + } + + return true + case BlockDetectionEmptyAnswer: + return answer.Rcode == dns.RcodeNameError && len(answer.Ns) == 0 && len(answer.Answer) == 0 && len(answer.Extra) == 0 + } + + return false +} diff --git a/resolver/config.go b/resolver/config.go index d4c4828d..ac9567b0 100644 --- a/resolver/config.go +++ b/resolver/config.go @@ -29,28 +29,30 @@ var ( // We encourage everyone who has the technical abilities to set their own preferred servers. // Default 1: Cloudflare - "dot://1.1.1.1:853?verify=cloudflare-dns.com", // Cloudflare - "dot://1.0.0.1:853?verify=cloudflare-dns.com", // Cloudflare + "dot://1.1.1.1:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip", // Cloudflare + "dot://1.0.0.1:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip", // Cloudflare // Default 2: Quad9 - "dot://9.9.9.9:853?verify=dns.quad9.net", // Quad9 - "dot://149.112.112.112:853?verify=dns.quad9.net", // Quad9 + "dot://9.9.9.9:853?verify=dns.quad9.net&name=Quad9&blockedif=empty", // Quad9 + "dot://149.112.112.112:853?verify=dns.quad9.net&name=Quad9&blockedif=empty", // Quad9 // Fallback 1: Cloudflare - "dns://1.1.1.1:53", // Cloudflare - "dns://1.0.0.1:53", // Cloudflare + "dns://1.1.1.1:53?name=Cloudflare&blockedif=zeroip", // Cloudflare + "dns://1.0.0.1:53?name=Cloudflare&blockedif=zeroip", // Cloudflare // Fallback 2: Quad9 - "dns://9.9.9.9:53", // Quad9 - "dns://149.112.112.112:53", // Quad9 + "dns://9.9.9.9:53?name=Quad9&blockedif=empty", // Quad9 + "dns://149.112.112.112:53?name=Quad9&blockedif=empty", // Quad9 // supported parameters // - `verify=domain`: verify domain (dot only) // future parameters: // // - `name=name`: human readable name for resolver - // - `blockedif=baredns`: how to detect if the dns service blocked something - // - `baredns`: NXDomain result, but without any other record in any section + // - `blockedif=empty`: how to detect if the dns service blocked something + // - `empty`: NXDomain result, but without any other record in any section + // - `refused`: Request was refused + // - `zeroip`: Answer only contains zeroip } CfgOptionNameServersKey = "dns/nameservers" diff --git a/resolver/ipinfo.go b/resolver/ipinfo.go index 760b25f3..0ecf9766 100644 --- a/resolver/ipinfo.go +++ b/resolver/ipinfo.go @@ -16,13 +16,92 @@ var ( }) ) +// ResolvedDomain holds a Domain name and a list of +// CNAMES that have been resolved. +type ResolvedDomain struct { + // Domain is the domain as requested by the application. + Domain string + + // CNAMEs is a list of CNAMEs that have been resolved for + // Domain. + CNAMEs []string +} + +// String returns a string representation of ResolvedDomain including +// the CNAME chain. It implements fmt.Stringer +func (resolved *ResolvedDomain) String() string { + ret := resolved.Domain + cnames := "" + + if len(resolved.CNAMEs) > 0 { + cnames = " (-> " + strings.Join(resolved.CNAMEs, "->") + ")" + } + + return ret + cnames +} + +// ResolvedDomains is a helper type for operating on a slice +// of ResolvedDomain +type ResolvedDomains []ResolvedDomain + +// String returns a string representation of all domains joined +// to a single string. +func (rds ResolvedDomains) String() string { + domains := make([]string, len(rds)) + for idx, n := range rds { + domains[idx] = n.String() + } + return strings.Join(domains, " or ") +} + +// MostRecentDomain returns the most recent domain. +func (rds ResolvedDomains) MostRecentDomain() *ResolvedDomain { + if len(rds) == 0 { + return nil + } + // TODO(ppacher): we could also do that by using ResolvedAt() + mostRecent := rds[len(rds)-1] + return &mostRecent +} + // IPInfo represents various information about an IP. type IPInfo struct { record.Base sync.Mutex - IP string - Domains []string + // IP holds the acutal IP address. + IP string + + // Domains holds a list of domains that have been + // resolved to IP. This field is deprecated and should + // be removed. + // DEPRECATED: remove with alpha. + Domains []string `json:"Domains,omitempty"` + + // ResolvedDomain is a slice of domains that + // have been requested by various applications + // and have been resolved to IP. + ResolvedDomains ResolvedDomains +} + +// AddDomain adds a new resolved domain to ipi. +func (ipi *IPInfo) AddDomain(resolved ResolvedDomain) bool { + for idx, d := range ipi.ResolvedDomains { + if d.Domain == resolved.Domain { + if utils.StringSliceEqual(d.CNAMEs, resolved.CNAMEs) { + return false + } + + // we have a different CNAME chain now, remove the previous + // entry and add it at the end. + ipi.ResolvedDomains = append(ipi.ResolvedDomains[:idx], ipi.ResolvedDomains[idx+1:]...) + ipi.ResolvedDomains = append(ipi.ResolvedDomains, resolved) + return true + } + } + + ipi.ResolvedDomains = append(ipi.ResolvedDomains, resolved) + return true } func makeIPInfoKey(ip string) string { @@ -46,6 +125,19 @@ func GetIPInfo(ip string) (*IPInfo, error) { if err != nil { return nil, err } + + // Legacy support, + // DEPRECATED: remove with alpha + if len(new.Domains) > 0 && len(new.ResolvedDomains) == 0 { + for _, d := range new.Domains { + new.ResolvedDomains = append(new.ResolvedDomains, ResolvedDomain{ + Domain: d, + // rest is empty... + }) + } + new.Domains = nil // clean up so we remove it from the database + } + return new, nil } @@ -57,17 +149,6 @@ func GetIPInfo(ip string) (*IPInfo, error) { return new, nil } -// AddDomain adds a domain to the list and reports back if it was added, or was already present. -func (ipi *IPInfo) AddDomain(domain string) (added bool) { - ipi.Lock() - defer ipi.Unlock() - if !utils.StringInSlice(ipi.Domains, domain) { - ipi.Domains = append([]string{domain}, ipi.Domains...) - return true - } - return false -} - // Save saves the IPInfo record to the database. func (ipi *IPInfo) Save() error { ipi.Lock() @@ -75,17 +156,21 @@ func (ipi *IPInfo) Save() error { ipi.SetKey(makeIPInfoKey(ipi.IP)) } ipi.Unlock() - return ipInfoDatabase.Put(ipi) -} -// FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or " -func (ipi *IPInfo) FmtDomains() string { - return strings.Join(ipi.Domains, " or ") + // Legacy support + // Ensure we don't write new Domain fields into the + // database. + // DEPRECATED: remove with alpha + if len(ipi.Domains) > 0 { + ipi.Domains = nil + } + + return ipInfoDatabase.Put(ipi) } // FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or " func (ipi *IPInfo) String() string { ipi.Lock() defer ipi.Unlock() - return fmt.Sprintf("