github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Start intel package adjustments

Browse Source
This commit is contained in:
Daniel
2018-10-22 17:03:23 +02:00
parent b1cd19a8e8
commit bf55c1232d
14 changed files with 912 additions and 842 deletions
Download Patch File Download Diff File Expand all files Collapse all files

464
intel/resolve.go
View File

@@ -3,30 +3,19 @@
package intel
import (
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/miekg/dns"
"github.com/tevino/abool"
"github.com/Safing/safing-core/configuration"
"github.com/Safing/safing-core/database"
"github.com/Safing/safing-core/log"
"github.com/Safing/safing-core/network/environment"
"github.com/Safing/safing-core/network/netutils"
"github.com/Safing/portbase/database"
"github.com/Safing/portbase/log"
"github.com/Safing/portmaster/status"
)
// TODO: make resolver interface for http package
@@ -79,296 +68,8 @@ import (
// global -> local scopes, global
// special -> local scopes, local
type Resolver struct {
// static
Server string
ServerAddress string
IP *net.IP
Port uint16
Resolve func(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
Search *[]string
AllowedSecurityLevel int8
SkipFqdnBeforeInit string
HTTPClient *http.Client
Source string
// atomic
Initialized *abool.AtomicBool
InitLock sync.Mutex
LastFail *int64
Expires *int64
// must be locked
LockReason sync.Mutex
FailReason string
// TODO: add:
// Expiration (for server got from DHCP / ICMPv6)
// bootstrapping (first query is already sent, wait for it to either succeed or fail - think about http bootstrapping here!)
// expanded server info: type, server address, server port, options - so we do not have to parse this every time!
}
func (r *Resolver) String() string {
return r.Server
}
func (r *Resolver) Address() string {
return urlFormatAddress(r.IP, r.Port)
}
type Scope struct {
Domain string
Resolvers []*Resolver
}
var (
config = configuration.Get()
globalResolvers []*Resolver // all resolvers
localResolvers []*Resolver // all resolvers that are in site-local or link-local IP ranges
localScopes []Scope // list of scopes with a list of local resolvers that can resolve the scope
mDNSResolver *Resolver // holds a reference to the mDNS resolver
resolversLock sync.RWMutex
env = environment.NewInterface()
dupReqMap = make(map[string]*sync.Mutex)
dupReqLock sync.Mutex
)
func init() {
loadResolvers(false)
}
func indexOfResolver(server string, list []*Resolver) int {
for k, v := range list {
if v.Server == server {
return k
}
}
return -1
}
func indexOfScope(domain string, list *[]Scope) int {
for k, v := range *list {
if v.Domain == domain {
return k
}
}
return -1
}
func parseAddress(server string) (*net.IP, uint16, error) {
delimiter := strings.LastIndex(server, ":")
if delimiter < 0 {
return nil, 0, errors.New("port missing")
}
ip := net.ParseIP(strings.Trim(server[:delimiter], "[]"))
if ip == nil {
return nil, 0, errors.New("invalid IP address")
}
port, err := strconv.Atoi(server[delimiter+1:])
if err != nil || port < 1 || port > 65536 {
return nil, 0, errors.New("invalid port")
}
return &ip, uint16(port), nil
}
func urlFormatAddress(ip *net.IP, port uint16) string {
var address string
if ipv4 := ip.To4(); ipv4 != nil {
address = fmt.Sprintf("%s:%d", ipv4.String(), port)
} else {
address = fmt.Sprintf("[%s]:%d", ip.String(), port)
}
return address
}
func loadResolvers(resetResolvers bool) {
// TODO: what happens when a lot of processes want to reload at once? we do not need to run this multiple times in a short time frame.
resolversLock.Lock()
defer resolversLock.Unlock()
var newResolvers []*Resolver
configuredServersLoop:
for _, server := range config.DNSServers {
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue configuredServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
parts := strings.Split(server, "|")
if len(parts) < 2 {
log.Warningf("intel: invalid DNS server in config: %s (invalid format)", server)
continue configuredServersLoop
}
var lastFail int64
new := &Resolver{
Server: server,
ServerAddress: parts[1],
LastFail: &lastFail,
Source: "config",
Initialized: abool.NewBool(false),
}
ip, port, err := parseAddress(parts[1])
if err != nil {
new.IP = ip
new.Port = port
}
switch {
case strings.HasPrefix(server, "DNS|"):
new.Resolve = queryDNS
new.AllowedSecurityLevel = configuration.SecurityLevelFortress
case strings.HasPrefix(server, "DoH|"):
new.Resolve = queryDNSoverHTTPS
new.AllowedSecurityLevel = configuration.SecurityLevelFortress
new.SkipFqdnBeforeInit = dns.Fqdn(strings.Split(parts[1], ":")[0])
tls := &tls.Config{
// TODO: use custom random
// Rand: io.Reader,
}
tr := &http.Transport{
MaxIdleConnsPerHost: 100,
TLSClientConfig: tls,
// TODO: use custom resolver as of Go1.9
}
if len(parts) == 3 && strings.HasPrefix(parts[2], "df:") {
// activate domain fronting
tls.ServerName = parts[2][3:]
addDomainFronting(new.SkipFqdnBeforeInit, dns.Fqdn(tls.ServerName))
new.SkipFqdnBeforeInit = dns.Fqdn(tls.ServerName)
}
new.HTTPClient = &http.Client{Transport: tr}
default:
log.Warningf("intel: invalid DNS server in config: %s (not starting with a valid identifier)", server)
continue configuredServersLoop
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// add local resolvers
assignedNameservers := environment.Nameservers()
assignedServersLoop:
for _, nameserver := range assignedNameservers {
server := fmt.Sprintf("DNS|%s", urlFormatAddress(&nameserver.IP, 53))
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue assignedServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
var lastFail int64
new := &Resolver{
Server: server,
ServerAddress: urlFormatAddress(&nameserver.IP, 53),
IP: &nameserver.IP,
Port: 53,
LastFail: &lastFail,
Resolve: queryDNS,
AllowedSecurityLevel: configuration.SecurityLevelFortress,
Initialized: abool.NewBool(false),
Source: "dhcp",
}
if netutils.IPIsLocal(nameserver.IP) && len(nameserver.Search) > 0 {
// only allow searches for local resolvers
var newSearch []string
for _, value := range nameserver.Search {
newSearch = append(newSearch, fmt.Sprintf(".%s.", strings.Trim(value, ".")))
}
new.Search = &newSearch
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// save resolvers
globalResolvers = newResolvers
if len(globalResolvers) == 0 {
log.Criticalf("intel: no (valid) dns servers found in configuration and system")
}
// make list with local resolvers
localResolvers = make([]*Resolver, 0)
for _, resolver := range globalResolvers {
if resolver.IP != nil && netutils.IPIsLocal(*resolver.IP) {
localResolvers = append(localResolvers, resolver)
}
}
// add resolvers to every scope the cover
localScopes = make([]Scope, 0)
for _, resolver := range globalResolvers {
if resolver.Search != nil {
// add resolver to custom searches
for _, search := range *resolver.Search {
if search == "." {
continue
}
key := indexOfScope(search, &localScopes)
if key == -1 {
localScopes = append(localScopes, Scope{
Domain: search,
Resolvers: []*Resolver{resolver},
})
} else {
localScopes[key].Resolvers = append(localScopes[key].Resolvers, resolver)
}
}
}
}
// init mdns resolver
if mDNSResolver == nil {
cannotFail := int64(-1)
mDNSResolver = &Resolver{
Server: "mDNS",
Resolve: queryMulticastDNS,
AllowedSecurityLevel: config.DoNotUseMDNS.Level(),
Initialized: abool.NewBool(false),
Source: "static",
LastFail: &cannotFail,
}
}
// sort scopes by length
sort.Slice(localScopes,
func(i, j int) bool {
return len(localScopes[i].Domain) > len(localScopes[j].Domain)
},
)
log.Trace("intel: loaded global resolvers:")
for _, resolver := range globalResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded local resolvers:")
for _, resolver := range localResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded scopes:")
for _, scope := range localScopes {
var scopeServers []string
for _, resolver := range scope.Resolvers {
scopeServers = append(scopeServers, resolver.Server)
}
log.Tracef("intel: %s: %s", scope.Domain, strings.Join(scopeServers, ", "))
}
}
// Resolve resolves the given query for a domain and type and returns a RRCache object or nil, if the query failed.
func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
func Resolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache {
fqdn = dns.Fqdn(fqdn)
// use this to time how long it takes resolve this domain
@@ -381,9 +82,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
var err error
switch uint16(qtype) {
case dns.TypeA:
rr, err = dns.NewRR("localhost. 3600 IN A 127.0.0.1")
rr, err = dns.NewRR("localhost. 17 IN A 127.0.0.1")
case dns.TypeAAAA:
rr, err = dns.NewRR("localhost. 3600 IN AAAA ::1")
rr, err = dns.NewRR("localhost. 17 IN AAAA ::1")
default:
return nil
}
@@ -406,7 +107,7 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
return resolveAndCache(fqdn, qtype, securityLevel)
}
if rrCache.Expires <= time.Now().Unix() {
if rrCache.TTL <= time.Now().Unix() {
rrCache.requestingNew = true
go resolveAndCache(fqdn, qtype, securityLevel)
}
@@ -420,17 +121,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
return rrCache
}
func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
func resolveAndCache(fqdn string, qtype dns.Type, securityLevel uint8) (rrCache *RRCache) {
// log.Tracef("intel: resolving %s%s", fqdn, qtype.String())
rrCache, ok := checkDomainFronting(fqdn, qtype, securityLevel)
if ok {
if rrCache == nil {
return nil
}
return rrCache
}
// dedup requests
dupKey := fmt.Sprintf("%s%s", fqdn, qtype.String())
dupReqLock.Lock()
@@ -469,29 +162,29 @@ func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
// persist to database
rrCache.Clean(600)
rrCache.CreateWithType(fqdn, qtype)
rrCache.Save()
return rrCache
}
func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
func intelligentResolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache {
// TODO: handle being offline
// TODO: handle multiple network connections
if config.Changed() {
log.Info("intel: config changed, reloading resolvers")
loadResolvers(false)
} else if env.NetworkChanged() {
log.Info("intel: network changed, reloading resolvers")
loadResolvers(true)
}
config.RLock()
defer config.RUnlock()
// TODO: handle these in a separate goroutine
// if config.Changed() {
// log.Info("intel: config changed, reloading resolvers")
// loadResolvers(false)
// } else if env.NetworkChanged() {
// log.Info("intel: network changed, reloading resolvers")
// loadResolvers(true)
// }
resolversLock.RLock()
defer resolversLock.RUnlock()
lastFailBoundary := time.Now().Unix() - config.DNSServerRetryRate
lastFailBoundary := time.Now().Unix() - nameserverRetryRate()
preDottedFqdn := "." + fqdn
// resolve:
@@ -510,11 +203,14 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
}
}
// check config
if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) {
if doNotUseMulticastDNS(securityLevel) {
return nil
}
// try mdns
rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel)
rrCache, err := queryMulticastDNS(fqdn, qtype)
if err != nil {
log.Errorf("intel: failed to query mdns: %s", err)
}
return rrCache
}
@@ -533,15 +229,18 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
switch {
case strings.HasSuffix(preDottedFqdn, ".local."):
// check config
if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) {
if doNotUseMulticastDNS(securityLevel) {
return nil
}
// try mdns
rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel)
rrCache, err := queryMulticastDNS(fqdn, qtype)
if err != nil {
log.Errorf("intel: failed to query mdns: %s", err)
}
return rrCache
case domainInScopes(preDottedFqdn, specialScopes):
// check config
if config.DoNotForwardSpecialDomains.IsSetWithLevel(securityLevel) {
if doNotResolveSpecialDomains(securityLevel) {
return nil
}
// try local resolvers
@@ -568,15 +267,15 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
}
func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel int8) (*RRCache, bool) {
func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel uint8) (*RRCache, bool) {
// skip if not allowed in current security level
if resolver.AllowedSecurityLevel < config.SecurityLevel() || resolver.AllowedSecurityLevel < securityLevel {
log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, config.SecurityLevel(), securityLevel)
if resolver.AllowedSecurityLevel < status.CurrentSecurityLevel() || resolver.AllowedSecurityLevel < securityLevel {
log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel)
return nil, false
}
// skip if not security level denies assigned dns servers
if config.DoNotUseAssignedDNS.IsSetWithLevel(securityLevel) && resolver.Source == "dhcp" {
log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d (%d)", resolver, config.SecurityLevel(), securityLevel, int8(config.DoNotUseAssignedDNS))
if doNotUseAssignedNameservers(securityLevel) && resolver.Source == "dhcp" {
log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel)
return nil, false
}
// check if failed recently
@@ -606,7 +305,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype
}
// resolve
log.Tracef("intel: trying to resolve %s%s with %s", fqdn, qtype.String(), resolver.Server)
rrCache, err := resolver.Resolve(resolver, fqdn, qtype)
rrCache, err := query(resolver, fqdn, qtype)
if err != nil {
// check if failing is disabled
if atomic.LoadInt64(resolver.LastFail) == -1 {
@@ -625,7 +324,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype
return rrCache, true
}
func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) {
func query(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) {
q := new(dns.Msg)
q.SetQuestion(fqdn, uint16(qtype))
@@ -633,8 +332,7 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
var reply *dns.Msg
var err error
for i := 0; i < 5; i++ {
client := new(dns.Client)
reply, _, err = client.Exchange(q, resolver.ServerAddress)
reply, _, err = resolver.clientManager.getDNSClient().Exchange(q, resolver.ServerAddress)
if err != nil {
// TODO: handle special cases
@@ -655,6 +353,8 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
}
new := &RRCache{
Domain: fqdn,
Question: qtype,
Answer: reply.Answer,
Ns: reply.Ns,
Extra: reply.Extra,
@@ -663,85 +363,3 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
// TODO: check if reply.Answer is valid
return new, nil
}
type DnsOverHttpsReply struct {
Status uint32
Truncated bool `json:"TC"`
Answer []DohRR
Additional []DohRR
}
type DohRR struct {
Name string `json:"name"`
Qtype uint16 `json:"type"`
TTL uint32 `json:"TTL"`
Data string `json:"data"`
}
func queryDNSoverHTTPS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) {
// API documentation: https://developers.google.com/speed/public-dns/docs/dns-over-https
payload := url.Values{}
payload.Add("name", fqdn)
payload.Add("type", strconv.Itoa(int(qtype)))
payload.Add("edns_client_subnet", "0.0.0.0/0")
// TODO: add random - only use upper- and lower-case letters, digits, hyphen, period, underscore and tilde
// payload.Add("random_padding", "")
resp, err := resolver.HTTPClient.Get(fmt.Sprintf("https://%s/resolve?%s", resolver.ServerAddress, payload.Encode()))
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: http error: %s", fqdn, qtype.String(), err)
// TODO: handle special cases
// 1. connect: network is unreachable
// intel: resolver DoH|dns.google.com:443|df:www.google.com failed (resolving discovery-v4-4.syncthing.net.A failed: http error: Get https://dns.google.com:443/resolve?edns_client_subnet=0.0.0.0%2F0&name=discovery-v4-4.syncthing.net.&type=1: dial tcp [2a00:1450:4001:819::2004]:443: connect: network is unreachable), moving to next
// 2. timeout
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("resolving %s%s failed: request was unsuccessful, got code %d", fqdn, qtype.String(), resp.StatusCode)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: error reading response body: %s", fqdn, qtype.String(), err)
}
var reply DnsOverHttpsReply
err = json.Unmarshal(body, &reply)
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: error parsing response body: %s", fqdn, qtype.String(), err)
}
if reply.Status != 0 {
// this happens if there is a server error (e.g. DNSSEC fail), ignore for now
// TODO: do something more intelligent
}
new := new(RRCache)
// TODO: handle TXT records
for _, entry := range reply.Answer {
rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data))
if err != nil {
log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)
continue
}
new.Answer = append(new.Answer, rr)
}
for _, entry := range reply.Additional {
rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data))
if err != nil {
log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)
continue
}
new.Extra = append(new.Extra, rr)
}
return new, nil
}
// TODO: implement T-DNS: DNS over TCP/TLS
// server list: https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers