diff --git a/intel/clients.go b/intel/clients.go new file mode 100644 index 00000000..18b34b02 --- /dev/null +++ b/intel/clients.go @@ -0,0 +1,94 @@ +package intel + +import ( + "crypto/tls" + "sync" + "time" + + "github.com/miekg/dns" +) + +type clientManager struct { + dnsClient *dns.Client + factory func() *dns.Client + + lock sync.Mutex + refreshAfter time.Time + ttl time.Duration // force refresh of connection to reduce traceability +} + +// ref: https://godoc.org/github.com/miekg/dns#Client + +func newDNSClientManager(resolver *Resolver) *clientManager { + return &clientManager{ + ttl: -1 * time.Minute, + factory: func() *dns.Client { + return &dns.Client{ + Timeout: 5 * time.Second, + } + }, + } +} + +func newTCPClientManager(resolver *Resolver) *clientManager { + return &clientManager{ + ttl: -15 * time.Minute, + factory: func() *dns.Client { + return &dns.Client{ + Net: "tcp", + Timeout: 5 * time.Second, + } + }, + } +} + +func newTLSClientManager(resolver *Resolver) *clientManager { + return &clientManager{ + ttl: -15 * time.Minute, + factory: func() *dns.Client { + return &dns.Client{ + Net: "tcp-tls", + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: resolver.VerifyDomain, + // TODO: use custom random + // Rand: io.Reader, + }, + Timeout: 5 * time.Second, + } + }, + } +} + +func newHTTPSClientManager(resolver *Resolver) *clientManager { + return &clientManager{ + ttl: -15 * time.Minute, + factory: func() *dns.Client { + new := &dns.Client{ + Net: "https", + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + // TODO: use custom random + // Rand: io.Reader, + }, + Timeout: 5 * time.Second, + } + if resolver.VerifyDomain != "" { + new.TLSConfig.ServerName = resolver.VerifyDomain + } + return new + }, + } +} + +func (cm *clientManager) getDNSClient() *dns.Client { + cm.lock.Lock() + defer cm.lock.Unlock() + + if cm.dnsClient == nil || time.Now().After(cm.refreshAfter) { + cm.dnsClient = cm.factory() + cm.refreshAfter = time.Now().Add(cm.ttl) + } + + return cm.dnsClient +} diff --git a/intel/config.go b/intel/config.go new file mode 100644 index 00000000..41d48f6b --- /dev/null +++ b/intel/config.go @@ -0,0 +1,83 @@ +package intel + +import ( + "github.com/Safing/portbase/config" + "github.com/Safing/portmaster/status" +) + +var ( + configuredNameServers config.StringArrayOption + defaultNameServers = []string{ + "tls|1.1.1.1:853|cloudflare-dns.com", // Cloudflare + "tls|1.0.0.1:853|cloudflare-dns.com", // Cloudflare + "tls|9.9.9.9:853|dns.quad9.net", // Quad9 + // "https|cloudflare-dns.com/dns-query", // HTTPS still experimental + "dns|1.1.1.1:53", // Cloudflare + "dns|1.0.0.1:53", // Cloudflare + "dns|9.9.9.9:53", // Quad9 + } + + nameserverRetryRate config.IntOption + doNotUseMulticastDNS status.SecurityLevelOption + doNotUseAssignedNameservers status.SecurityLevelOption + doNotResolveSpecialDomains status.SecurityLevelOption +) + +func init() { + config.Register(&config.Option{ + Name: "Nameservers (DNS)", + Key: "intel/nameservers", + Description: "Nameserver to use for resolving DNS requests.", + ExpertiseLevel: config.ExpertiseLevelExpert, + OptType: config.OptTypeStringArray, + DefaultValue: defaultNameServers, + ValidationRegex: "^(dns|tcp|tls|https)$", + }) + configuredNameServers = config.Concurrent.GetAsStringArray("intel/nameservers", defaultNameServers) + + config.Register(&config.Option{ + Name: "Nameserver Retry Rate", + Key: "intel/nameserverRetryRate", + Description: "Rate at which to retry failed nameservers, in seconds.", + ExpertiseLevel: config.ExpertiseLevelExpert, + OptType: config.OptTypeInt, + DefaultValue: 600, + }) + nameserverRetryRate = config.Concurrent.GetAsInt("intel/nameserverRetryRate", 0) + + config.Register(&config.Option{ + Name: "Do not use Multicast DNS", + Key: "intel/doNotUseMulticastDNS", + Description: "", + ExpertiseLevel: config.ExpertiseLevelExpert, + OptType: config.OptTypeInt, + ExternalOptType: "security level", + DefaultValue: 3, + ValidationRegex: "^(1|2|3)$", + }) + doNotUseMulticastDNS = status.ConfigIsActiveConcurrent("intel/doNotUseMulticastDNS") + + config.Register(&config.Option{ + Name: "Do not use assigned Nameservers", + Key: "intel/doNotUseAssignedNameservers", + Description: "that were acquired by the network (dhcp) or system", + ExpertiseLevel: config.ExpertiseLevelExpert, + OptType: config.OptTypeInt, + ExternalOptType: "security level", + DefaultValue: 3, + ValidationRegex: "^(1|2|3)$", + }) + doNotUseAssignedNameservers = status.ConfigIsActiveConcurrent("intel/doNotUseAssignedNameservers") + + config.Register(&config.Option{ + Name: "Do not resolve special domains", + Key: "intel/doNotResolveSpecialDomains", + Description: "Do not resolve special (top level) domains: example, example.com, example.net, example.org, invalid, test, onion. (RFC6761, RFC7686)", + ExpertiseLevel: config.ExpertiseLevelExpert, + OptType: config.OptTypeInt, + ExternalOptType: "security level", + DefaultValue: 3, + ValidationRegex: "^(1|2|3)$", + }) + doNotResolveSpecialDomains = status.ConfigIsActiveConcurrent("intel/doNotResolveSpecialDomains") +} diff --git a/intel/data.go b/intel/data.go deleted file mode 100644 index 67074f22..00000000 --- a/intel/data.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - -package intel - -import ( - "github.com/Safing/safing-core/database" - - datastore "github.com/ipfs/go-datastore" -) - -// EntityClassification holds classification information about an internet entity. -type EntityClassification struct { - lists []byte -} - -// Intel holds intelligence data for a domain. -type Intel struct { - database.Base - Domain string - DomainOwner string - CertOwner string - Classification *EntityClassification -} - -var intelModel *Intel // only use this as parameter for database.EnsureModel-like functions - -func init() { - database.RegisterModel(intelModel, func() database.Model { return new(Intel) }) -} - -// Create saves the Intel with the provided name in the default namespace. -func (m *Intel) Create(name string) error { - return m.CreateObject(&database.IntelCache, name, m) -} - -// CreateInNamespace saves the Intel with the provided name in the provided namespace. -func (m *Intel) CreateInNamespace(namespace *datastore.Key, name string) error { - return m.CreateObject(namespace, name, m) -} - -// Save saves the Intel. -func (m *Intel) Save() error { - return m.SaveObject(m) -} - -// getIntel fetches the Intel with the provided name in the default namespace. -func getIntel(name string) (*Intel, error) { - return getIntelFromNamespace(&database.IntelCache, name) -} - -// getIntelFromNamespace fetches the Intel with the provided name in the provided namespace. -func getIntelFromNamespace(namespace *datastore.Key, name string) (*Intel, error) { - object, err := database.GetAndEnsureModel(namespace, name, intelModel) - if err != nil { - return nil, err - } - model, ok := object.(*Intel) - if !ok { - return nil, database.NewMismatchError(object, intelModel) - } - return model, nil -} diff --git a/intel/dns.go b/intel/dns.go deleted file mode 100644 index 2d15f13b..00000000 --- a/intel/dns.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - -package intel - -import ( - "fmt" - "net" - "time" - - "github.com/Safing/safing-core/database" - - datastore "github.com/ipfs/go-datastore" - "github.com/miekg/dns" -) - -// RRCache is used to cache DNS data -type RRCache struct { - Answer []dns.RR - Ns []dns.RR - Extra []dns.RR - Expires int64 - Modified int64 - servedFromCache bool - requestingNew bool -} - -func (m *RRCache) Clean(minExpires uint32) { - - var lowestTTL uint32 = 0xFFFFFFFF - var header *dns.RR_Header - - // set TTLs to 17 - // TODO: double append? is there something more elegant? - for _, rr := range append(m.Answer, append(m.Ns, m.Extra...)...) { - header = rr.Header() - if lowestTTL > header.Ttl { - lowestTTL = header.Ttl - } - header.Ttl = 17 - } - - // TTL must be at least minExpires - if lowestTTL < minExpires { - lowestTTL = minExpires - } - - m.Expires = time.Now().Unix() + int64(lowestTTL) - m.Modified = time.Now().Unix() - -} - -func (m *RRCache) ExportAllARecords() (ips []net.IP) { - for _, rr := range m.Answer { - if rr.Header().Class == dns.ClassINET && rr.Header().Rrtype == dns.TypeA { - aRecord, ok := rr.(*dns.A) - if ok { - ips = append(ips, aRecord.A) - } - } else if rr.Header().Class == dns.ClassINET && rr.Header().Rrtype == dns.TypeAAAA { - aRecord, ok := rr.(*dns.AAAA) - if ok { - ips = append(ips, aRecord.AAAA) - } - } - } - return -} - -func (m *RRCache) ToRRSave() *RRSave { - var s RRSave - s.Expires = m.Expires - s.Modified = m.Modified - for _, entry := range m.Answer { - s.Answer = append(s.Answer, entry.String()) - } - for _, entry := range m.Ns { - s.Ns = append(s.Ns, entry.String()) - } - for _, entry := range m.Extra { - s.Extra = append(s.Extra, entry.String()) - } - return &s -} - -func (m *RRCache) Create(name string) error { - s := m.ToRRSave() - return s.CreateObject(&database.DNSCache, name, s) -} - -func (m *RRCache) CreateWithType(name string, qtype dns.Type) error { - s := m.ToRRSave() - return s.Create(fmt.Sprintf("%s%s", name, qtype.String())) -} - -func (m *RRCache) Save() error { - s := m.ToRRSave() - return s.SaveObject(s) -} - -func GetRRCache(domain string, qtype dns.Type) (*RRCache, error) { - return GetRRCacheFromNamespace(&database.DNSCache, domain, qtype) -} - -func GetRRCacheFromNamespace(namespace *datastore.Key, domain string, qtype dns.Type) (*RRCache, error) { - var m RRCache - - rrSave, err := GetRRSaveFromNamespace(namespace, domain, qtype) - if err != nil { - return nil, err - } - - m.Expires = rrSave.Expires - m.Modified = rrSave.Modified - for _, entry := range rrSave.Answer { - rr, err := dns.NewRR(entry) - if err == nil { - m.Answer = append(m.Answer, rr) - } - } - for _, entry := range rrSave.Ns { - rr, err := dns.NewRR(entry) - if err == nil { - m.Ns = append(m.Ns, rr) - } - } - for _, entry := range rrSave.Extra { - rr, err := dns.NewRR(entry) - if err == nil { - m.Extra = append(m.Extra, rr) - } - } - - m.servedFromCache = true - return &m, nil -} - -// ServedFromCache marks the RRCache as served from cache. -func (m *RRCache) ServedFromCache() bool { - return m.servedFromCache -} - -// RequestingNew informs that it has expired and new RRs are being fetched. -func (m *RRCache) RequestingNew() bool { - return m.requestingNew -} - -// Flags formats ServedFromCache and RequestingNew to a condensed, flag-like format. -func (m *RRCache) Flags() string { - switch { - case m.servedFromCache && m.requestingNew: - return " [CR]" - case m.servedFromCache: - return " [C]" - case m.requestingNew: - return " [R]" // theoretically impossible, but let's leave it here, just in case - default: - return "" - } -} - -// IsNXDomain returnes whether the result is nxdomain. -func (m *RRCache) IsNXDomain() bool { - return len(m.Answer) == 0 -} - -// RRSave is helper struct to RRCache to better save data to the database. -type RRSave struct { - database.Base - Answer []string - Ns []string - Extra []string - Expires int64 - Modified int64 -} - -var rrSaveModel *RRSave // only use this as parameter for database.EnsureModel-like functions - -func init() { - database.RegisterModel(rrSaveModel, func() database.Model { return new(RRSave) }) -} - -// Create saves RRSave with the provided name in the default namespace. -func (m *RRSave) Create(name string) error { - return m.CreateObject(&database.DNSCache, name, m) -} - -// CreateWithType saves RRSave with the provided name and type in the default namespace. -func (m *RRSave) CreateWithType(name string, qtype dns.Type) error { - return m.Create(fmt.Sprintf("%s%s", name, qtype.String())) -} - -// CreateInNamespace saves RRSave with the provided name in the provided namespace. -func (m *RRSave) CreateInNamespace(namespace *datastore.Key, name string) error { - return m.CreateObject(namespace, name, m) -} - -// Save saves RRSave. -func (m *RRSave) Save() error { - return m.SaveObject(m) -} - -// GetRRSave fetches RRSave with the provided name in the default namespace. -func GetRRSave(name string, qtype dns.Type) (*RRSave, error) { - return GetRRSaveFromNamespace(&database.DNSCache, name, qtype) -} - -// GetRRSaveFromNamespace fetches RRSave with the provided name in the provided namespace. -func GetRRSaveFromNamespace(namespace *datastore.Key, name string, qtype dns.Type) (*RRSave, error) { - object, err := database.GetAndEnsureModel(namespace, fmt.Sprintf("%s%s", name, qtype.String()), rrSaveModel) - if err != nil { - return nil, err - } - model, ok := object.(*RRSave) - if !ok { - return nil, database.NewMismatchError(object, rrSaveModel) - } - return model, nil -} diff --git a/intel/domainfronting.go b/intel/domainfronting.go deleted file mode 100644 index 516fc0d1..00000000 --- a/intel/domainfronting.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - -package intel - -import ( - "github.com/Safing/safing-core/log" - "sync" - - "github.com/miekg/dns" -) - -var ( - dfMap = make(map[string]string) - dfMapLock sync.RWMutex -) - -func checkDomainFronting(hidden string, qtype dns.Type, securityLevel int8) (*RRCache, bool) { - dfMapLock.RLock() - front, ok := dfMap[hidden] - dfMapLock.RUnlock() - if !ok { - return nil, false - } - log.Tracef("intel: applying domain fronting %s -> %s", hidden, front) - // get domain name - rrCache := resolveAndCache(front, qtype, securityLevel) - if rrCache == nil { - return nil, true - } - // replace domain name - var header *dns.RR_Header - for _, rr := range rrCache.Answer { - header = rr.Header() - if header.Name == front { - header.Name = hidden - } - } - // save under front - rrCache.CreateWithType(hidden, qtype) - return rrCache, true -} - -func addDomainFronting(hidden string, front string) { - dfMapLock.Lock() - dfMap[hidden] = front - dfMapLock.Unlock() - return -} diff --git a/intel/intel.go b/intel/intel.go index b6bf3f3c..5ed4edd3 100644 --- a/intel/intel.go +++ b/intel/intel.go @@ -3,44 +3,66 @@ package intel import ( - "github.com/Safing/safing-core/database" - "github.com/Safing/safing-core/modules" + "fmt" + "sync" - "github.com/miekg/dns" + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/database/record" ) var ( - intelModule *modules.Module + intelDatabase = database.NewInterface(&database.Options{ + AlwaysSetRelativateExpiry: 2592000, // 30 days + }) ) -func init() { - intelModule = modules.Register("Intel", 128) - go Start() +// Intel holds intelligence data for a domain. +type Intel struct { + record.Base + sync.Mutex + + Domain string } -// GetIntel returns an Intel object of the given domain. The returned Intel object MUST not be modified. -func GetIntel(domain string) *Intel { - fqdn := dns.Fqdn(domain) - intel, err := getIntel(fqdn) +func makeIntelKey(domain string) string { + return fmt.Sprintf("intel:Intel/%s", domain) +} + +// GetIntelFromDB gets an Intel record from the database. +func GetIntelFromDB(domain string) (*Intel, error) { + key := makeIntelKey(domain) + + r, err := intelDatabase.Get(key) if err != nil { - if err == database.ErrNotFound { - intel = &Intel{Domain: fqdn} - intel.Create(fqdn) - } else { - return nil - } + return nil, err } - return intel + + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + new := &Intel{} + err = record.Unwrap(r, new) + if err != nil { + return nil, err + } + return new, nil + } + + // or adjust type + new, ok := r.(*Intel) + if !ok { + return nil, fmt.Errorf("record not of type *Intel, but %T", r) + } + return new, nil } -func GetIntelAndRRs(domain string, qtype dns.Type, securityLevel int8) (intel *Intel, rrs *RRCache) { - intel = GetIntel(domain) - rrs = Resolve(domain, qtype, securityLevel) - return +// Save saves the Intel record to the database. +func (intel *Intel) Save() error { + intel.SetKey(makeIntelKey(intel.Domain)) + return intelDatabase.PutNew(intel) } -func Start() { - // mocking until intel has its own goroutines - defer intelModule.StopComplete() - <-intelModule.Stop +// GetIntel fetches intelligence data for the given domain. +func GetIntel(domain string) (*Intel, error) { + return &Intel{Domain: domain}, nil } diff --git a/intel/ipinfo.go b/intel/ipinfo.go index 007e67c3..c375e6bf 100644 --- a/intel/ipinfo.go +++ b/intel/ipinfo.go @@ -1,61 +1,69 @@ -// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. - package intel import ( + "fmt" + "net" "strings" + "sync" - "github.com/Safing/safing-core/database" + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/database/record" +) - datastore "github.com/ipfs/go-datastore" +var ( + ipInfoDatabase = database.NewInterface(&database.Options{ + AlwaysSetRelativateExpiry: 86400, // 24 hours + }) ) // IPInfo represents various information about an IP. type IPInfo struct { - database.Base + record.Base + sync.Mutex + + IP net.IP Domains []string } -var ipInfoModel *IPInfo // only use this as parameter for database.EnsureModel-like functions - -func init() { - database.RegisterModel(ipInfoModel, func() database.Model { return new(IPInfo) }) +func makeIPInfoKey(ip net.IP) string { + return fmt.Sprintf("intel:IPInfo/%s", ip.String()) } -// Create saves the IPInfo with the provided name in the default namespace. -func (m *IPInfo) Create(name string) error { - return m.CreateObject(&database.IPInfoCache, name, m) -} +// GetIPInfo gets an IPInfo record from the database. +func GetIPInfo(ip net.IP) (*IPInfo, error) { + key := makeIPInfoKey(ip) -// CreateInNamespace saves the IPInfo with the provided name in the provided namespace. -func (m *IPInfo) CreateInNamespace(namespace *datastore.Key, name string) error { - return m.CreateObject(namespace, name, m) -} - -// Save saves the IPInfo. -func (m *IPInfo) Save() error { - return m.SaveObject(m) -} - -// GetIPInfo fetches the IPInfo with the provided name in the default namespace. -func GetIPInfo(name string) (*IPInfo, error) { - return GetIPInfoFromNamespace(&database.IPInfoCache, name) -} - -// GetIPInfoFromNamespace fetches the IPInfo with the provided name in the provided namespace. -func GetIPInfoFromNamespace(namespace *datastore.Key, name string) (*IPInfo, error) { - object, err := database.GetAndEnsureModel(namespace, name, ipInfoModel) + r, err := ipInfoDatabase.Get(key) if err != nil { return nil, err } - model, ok := object.(*IPInfo) - if !ok { - return nil, database.NewMismatchError(object, ipInfoModel) + + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + new := &IPInfo{} + err = record.Unwrap(r, new) + if err != nil { + return nil, err + } + return new, nil } - return model, nil + + // or adjust type + new, ok := r.(*IPInfo) + if !ok { + return nil, fmt.Errorf("record not of type *IPInfo, but %T", r) + } + return new, nil +} + +// Save saves the IPInfo record to the database. +func (ipi *IPInfo) Save() error { + ipi.SetKey(makeIPInfoKey(ipi.IP)) + return ipInfoDatabase.PutNew(ipi) } // FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or " -func (m *IPInfo) FmtDomains() string { - return strings.Join(m.Domains, " or ") +func (ipi *IPInfo) FmtDomains() string { + return strings.Join(ipi.Domains, " or ") } diff --git a/intel/main.go b/intel/main.go new file mode 100644 index 00000000..dd4a7d66 --- /dev/null +++ b/intel/main.go @@ -0,0 +1,40 @@ +package intel + +import ( + "github.com/miekg/dns" + + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/log" + "github.com/Safing/portbase/modules" +) + +func init() { + modules.Register("intel", nil, start, nil, "database") +} + +func start() error { + _, err := database.Register(&database.Database{ + Name: "intel", + Description: "Intelligence and DNS Data", + StorageType: "badger", + PrimaryAPI: "", + }) + if err != nil { + return err + } + + // load resolvers from config and environment + loadResolvers(false) + + return nil +} + +func GetIntelAndRRs(domain string, qtype dns.Type, securityLevel uint8) (intel *Intel, rrs *RRCache) { + intel, err := GetIntel(domain) + if err != nil { + log.Errorf("intel: failed to get intel: %s", err) + intel = nil + } + rrs = Resolve(domain, qtype, securityLevel) + return +} diff --git a/intel/mdns.go b/intel/mdns.go index 103688de..67491b7c 100644 --- a/intel/mdns.go +++ b/intel/mdns.go @@ -6,12 +6,13 @@ import ( "errors" "fmt" "net" - "github.com/Safing/safing-core/log" "strings" "sync" "time" "github.com/miekg/dns" + + "github.com/Safing/portbase/log" ) const ( @@ -89,7 +90,7 @@ func listenToMDNS() { var question *dns.Question var saveFullRequest bool - scavengedRecords := make(map[string]*dns.RR) + scavengedRecords := make(map[string]dns.RR) var rrCache *RRCache // save every received response @@ -114,7 +115,7 @@ func listenToMDNS() { continue } - // continue if no question + // get question, some servers do not reply with question if len(message.Question) == 0 { questionsLock.Lock() savedQ, ok := questions[message.MsgHdr.Id] @@ -138,8 +139,11 @@ func listenToMDNS() { // get entry from database if saveFullRequest { rrCache, err = GetRRCache(question.Name, dns.Type(question.Qtype)) - if err != nil || rrCache.Modified < time.Now().Add(-2*time.Second).Unix() || rrCache.Expires < time.Now().Unix() { - rrCache = &RRCache{} + if err != nil || rrCache.updated < time.Now().Add(-2*time.Second).Unix() || rrCache.TTL < time.Now().Unix() { + rrCache = &RRCache{ + Domain: question.Name, + Question: dns.Type(question.Qtype), + } } } @@ -155,12 +159,12 @@ func listenToMDNS() { } switch entry.(type) { case *dns.A: - scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = entry case *dns.AAAA: - scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = entry case *dns.PTR: if !strings.HasPrefix(entry.Header().Name, "_") { - scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = entry } } } @@ -177,17 +181,16 @@ func listenToMDNS() { } switch entry.(type) { case *dns.A: - scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%s_A", entry.Header().Name)] = entry case *dns.AAAA: - scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%s_AAAA", entry.Header().Name)] = entry case *dns.PTR: if !strings.HasPrefix(entry.Header().Name, "_") { - scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%s_PTR", entry.Header().Name)] = entry } } } } - // TODO: scan Extra for A and AAAA records and save them seperately for _, entry := range message.Extra { if strings.HasSuffix(entry.Header().Name, ".local.") || domainInScopes(entry.Header().Name, localReverseScopes) { if saveFullRequest { @@ -200,34 +203,35 @@ func listenToMDNS() { } switch entry.(type) { case *dns.A: - scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = entry case *dns.AAAA: - scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = entry case *dns.PTR: if !strings.HasPrefix(entry.Header().Name, "_") { - scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry + scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = entry } } } } + var questionID string if saveFullRequest { rrCache.Clean(60) - rrCache.CreateWithType(question.Name, dns.Type(question.Qtype)) - // log.Tracef("intel: mdns saved full reply to %s%s", question.Name, dns.Type(question.Qtype).String()) + rrCache.Save() + questionID = fmt.Sprintf("%s%s", question.Name, dns.Type(question.Qtype).String()) } for k, v := range scavengedRecords { - if saveFullRequest { - if k == fmt.Sprintf("%s%s", question.Name, dns.Type(question.Qtype).String()) { - continue - } + if saveFullRequest && k == questionID { + continue } rrCache = &RRCache{ - Answer: []dns.RR{*v}, + Domain: v.Header().Name, + Question: dns.Type(v.Header().Class), + Answer: []dns.RR{v}, } rrCache.Clean(60) - rrCache.Create(k) + rrCache.Save() // log.Tracef("intel: mdns scavenged %s", k) } @@ -261,7 +265,7 @@ func listenForDNSPackets(conn *net.UDPConn, messages chan *dns.Msg) { } } -func queryMulticastDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { +func queryMulticastDNS(fqdn string, qtype dns.Type) (*RRCache, error) { q := new(dns.Msg) q.SetQuestion(fqdn, uint16(qtype)) // request unicast response diff --git a/intel/namerecord.go b/intel/namerecord.go new file mode 100644 index 00000000..ea9b52e3 --- /dev/null +++ b/intel/namerecord.go @@ -0,0 +1,72 @@ +package intel + +import ( + "errors" + "fmt" + "sync" + + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/database/record" +) + +var ( + recordDatabase = database.NewInterface(&database.Options{ + AlwaysSetRelativateExpiry: 2592000, // 30 days + CacheSize: 100, + }) +) + +// NameRecord is helper struct to RRCache to better save data to the database. +type NameRecord struct { + record.Base + sync.Mutex + + Domain string + Question string + Answer []string + Ns []string + Extra []string + TTL int64 +} + +func makeNameRecordKey(domain string, question string) string { + return fmt.Sprintf("intel:NameRecords/%s%s", domain, question) +} + +// GetNameRecord gets a NameRecord from the database. +func GetNameRecord(domain string, question string) (*NameRecord, error) { + key := makeNameRecordKey(domain, question) + + r, err := recordDatabase.Get(key) + if err != nil { + return nil, err + } + + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + new := &NameRecord{} + err = record.Unwrap(r, new) + if err != nil { + return nil, err + } + return new, nil + } + + // or adjust type + new, ok := r.(*NameRecord) + if !ok { + return nil, fmt.Errorf("record not of type *NameRecord, but %T", r) + } + return new, nil +} + +// Save saves the NameRecord to the database. +func (rec *NameRecord) Save() error { + if rec.Domain == "" || rec.Question == "" { + return errors.New("could not save NameRecord, missing Domain and/or Question") + } + + rec.SetKey(makeNameRecordKey(rec.Domain, rec.Question)) + return recordDatabase.PutNew(rec) +} diff --git a/intel/resolve.go b/intel/resolve.go index 786bd46b..6febbac3 100644 --- a/intel/resolve.go +++ b/intel/resolve.go @@ -3,30 +3,19 @@ package intel import ( - "crypto/tls" - "encoding/json" - "errors" "fmt" - "io/ioutil" "math/rand" "net" - "net/http" - "net/url" - "sort" - "strconv" "strings" "sync" "sync/atomic" "time" "github.com/miekg/dns" - "github.com/tevino/abool" - "github.com/Safing/safing-core/configuration" - "github.com/Safing/safing-core/database" - "github.com/Safing/safing-core/log" - "github.com/Safing/safing-core/network/environment" - "github.com/Safing/safing-core/network/netutils" + "github.com/Safing/portbase/database" + "github.com/Safing/portbase/log" + "github.com/Safing/portmaster/status" ) // TODO: make resolver interface for http package @@ -79,296 +68,8 @@ import ( // global -> local scopes, global // special -> local scopes, local -type Resolver struct { - // static - Server string - ServerAddress string - IP *net.IP - Port uint16 - Resolve func(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) - Search *[]string - AllowedSecurityLevel int8 - SkipFqdnBeforeInit string - HTTPClient *http.Client - Source string - - // atomic - Initialized *abool.AtomicBool - InitLock sync.Mutex - LastFail *int64 - Expires *int64 - - // must be locked - LockReason sync.Mutex - FailReason string - - // TODO: add: - // Expiration (for server got from DHCP / ICMPv6) - // bootstrapping (first query is already sent, wait for it to either succeed or fail - think about http bootstrapping here!) - // expanded server info: type, server address, server port, options - so we do not have to parse this every time! -} - -func (r *Resolver) String() string { - return r.Server -} - -func (r *Resolver) Address() string { - return urlFormatAddress(r.IP, r.Port) -} - -type Scope struct { - Domain string - Resolvers []*Resolver -} - -var ( - config = configuration.Get() - - globalResolvers []*Resolver // all resolvers - localResolvers []*Resolver // all resolvers that are in site-local or link-local IP ranges - localScopes []Scope // list of scopes with a list of local resolvers that can resolve the scope - mDNSResolver *Resolver // holds a reference to the mDNS resolver - resolversLock sync.RWMutex - - env = environment.NewInterface() - - dupReqMap = make(map[string]*sync.Mutex) - dupReqLock sync.Mutex -) - -func init() { - loadResolvers(false) -} - -func indexOfResolver(server string, list []*Resolver) int { - for k, v := range list { - if v.Server == server { - return k - } - } - return -1 -} - -func indexOfScope(domain string, list *[]Scope) int { - for k, v := range *list { - if v.Domain == domain { - return k - } - } - return -1 -} - -func parseAddress(server string) (*net.IP, uint16, error) { - delimiter := strings.LastIndex(server, ":") - if delimiter < 0 { - return nil, 0, errors.New("port missing") - } - ip := net.ParseIP(strings.Trim(server[:delimiter], "[]")) - if ip == nil { - return nil, 0, errors.New("invalid IP address") - } - port, err := strconv.Atoi(server[delimiter+1:]) - if err != nil || port < 1 || port > 65536 { - return nil, 0, errors.New("invalid port") - } - return &ip, uint16(port), nil -} - -func urlFormatAddress(ip *net.IP, port uint16) string { - var address string - if ipv4 := ip.To4(); ipv4 != nil { - address = fmt.Sprintf("%s:%d", ipv4.String(), port) - } else { - address = fmt.Sprintf("[%s]:%d", ip.String(), port) - } - return address -} - -func loadResolvers(resetResolvers bool) { - // TODO: what happens when a lot of processes want to reload at once? we do not need to run this multiple times in a short time frame. - resolversLock.Lock() - defer resolversLock.Unlock() - - var newResolvers []*Resolver - -configuredServersLoop: - for _, server := range config.DNSServers { - key := indexOfResolver(server, newResolvers) - if key >= 0 { - continue configuredServersLoop - } - key = indexOfResolver(server, globalResolvers) - if resetResolvers || key == -1 { - parts := strings.Split(server, "|") - if len(parts) < 2 { - log.Warningf("intel: invalid DNS server in config: %s (invalid format)", server) - continue configuredServersLoop - } - var lastFail int64 - new := &Resolver{ - Server: server, - ServerAddress: parts[1], - LastFail: &lastFail, - Source: "config", - Initialized: abool.NewBool(false), - } - ip, port, err := parseAddress(parts[1]) - if err != nil { - new.IP = ip - new.Port = port - } - switch { - case strings.HasPrefix(server, "DNS|"): - new.Resolve = queryDNS - new.AllowedSecurityLevel = configuration.SecurityLevelFortress - case strings.HasPrefix(server, "DoH|"): - new.Resolve = queryDNSoverHTTPS - new.AllowedSecurityLevel = configuration.SecurityLevelFortress - new.SkipFqdnBeforeInit = dns.Fqdn(strings.Split(parts[1], ":")[0]) - - tls := &tls.Config{ - // TODO: use custom random - // Rand: io.Reader, - } - tr := &http.Transport{ - MaxIdleConnsPerHost: 100, - TLSClientConfig: tls, - // TODO: use custom resolver as of Go1.9 - } - if len(parts) == 3 && strings.HasPrefix(parts[2], "df:") { - // activate domain fronting - tls.ServerName = parts[2][3:] - addDomainFronting(new.SkipFqdnBeforeInit, dns.Fqdn(tls.ServerName)) - new.SkipFqdnBeforeInit = dns.Fqdn(tls.ServerName) - } - new.HTTPClient = &http.Client{Transport: tr} - - default: - log.Warningf("intel: invalid DNS server in config: %s (not starting with a valid identifier)", server) - continue configuredServersLoop - } - newResolvers = append(newResolvers, new) - } else { - newResolvers = append(newResolvers, globalResolvers[key]) - } - } - - // add local resolvers - assignedNameservers := environment.Nameservers() -assignedServersLoop: - for _, nameserver := range assignedNameservers { - server := fmt.Sprintf("DNS|%s", urlFormatAddress(&nameserver.IP, 53)) - key := indexOfResolver(server, newResolvers) - if key >= 0 { - continue assignedServersLoop - } - key = indexOfResolver(server, globalResolvers) - if resetResolvers || key == -1 { - var lastFail int64 - new := &Resolver{ - Server: server, - ServerAddress: urlFormatAddress(&nameserver.IP, 53), - IP: &nameserver.IP, - Port: 53, - LastFail: &lastFail, - Resolve: queryDNS, - AllowedSecurityLevel: configuration.SecurityLevelFortress, - Initialized: abool.NewBool(false), - Source: "dhcp", - } - if netutils.IPIsLocal(nameserver.IP) && len(nameserver.Search) > 0 { - // only allow searches for local resolvers - var newSearch []string - for _, value := range nameserver.Search { - newSearch = append(newSearch, fmt.Sprintf(".%s.", strings.Trim(value, "."))) - } - new.Search = &newSearch - } - newResolvers = append(newResolvers, new) - } else { - newResolvers = append(newResolvers, globalResolvers[key]) - } - } - - // save resolvers - globalResolvers = newResolvers - if len(globalResolvers) == 0 { - log.Criticalf("intel: no (valid) dns servers found in configuration and system") - } - - // make list with local resolvers - localResolvers = make([]*Resolver, 0) - for _, resolver := range globalResolvers { - if resolver.IP != nil && netutils.IPIsLocal(*resolver.IP) { - localResolvers = append(localResolvers, resolver) - } - } - - // add resolvers to every scope the cover - localScopes = make([]Scope, 0) - for _, resolver := range globalResolvers { - - if resolver.Search != nil { - // add resolver to custom searches - for _, search := range *resolver.Search { - if search == "." { - continue - } - key := indexOfScope(search, &localScopes) - if key == -1 { - localScopes = append(localScopes, Scope{ - Domain: search, - Resolvers: []*Resolver{resolver}, - }) - } else { - localScopes[key].Resolvers = append(localScopes[key].Resolvers, resolver) - } - } - - } - } - - // init mdns resolver - if mDNSResolver == nil { - cannotFail := int64(-1) - mDNSResolver = &Resolver{ - Server: "mDNS", - Resolve: queryMulticastDNS, - AllowedSecurityLevel: config.DoNotUseMDNS.Level(), - Initialized: abool.NewBool(false), - Source: "static", - LastFail: &cannotFail, - } - } - - // sort scopes by length - sort.Slice(localScopes, - func(i, j int) bool { - return len(localScopes[i].Domain) > len(localScopes[j].Domain) - }, - ) - - log.Trace("intel: loaded global resolvers:") - for _, resolver := range globalResolvers { - log.Tracef("intel: %s", resolver.Server) - } - log.Trace("intel: loaded local resolvers:") - for _, resolver := range localResolvers { - log.Tracef("intel: %s", resolver.Server) - } - log.Trace("intel: loaded scopes:") - for _, scope := range localScopes { - var scopeServers []string - for _, resolver := range scope.Resolvers { - scopeServers = append(scopeServers, resolver.Server) - } - log.Tracef("intel: %s: %s", scope.Domain, strings.Join(scopeServers, ", ")) - } - -} - // Resolve resolves the given query for a domain and type and returns a RRCache object or nil, if the query failed. -func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { +func Resolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache { fqdn = dns.Fqdn(fqdn) // use this to time how long it takes resolve this domain @@ -381,9 +82,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { var err error switch uint16(qtype) { case dns.TypeA: - rr, err = dns.NewRR("localhost. 3600 IN A 127.0.0.1") + rr, err = dns.NewRR("localhost. 17 IN A 127.0.0.1") case dns.TypeAAAA: - rr, err = dns.NewRR("localhost. 3600 IN AAAA ::1") + rr, err = dns.NewRR("localhost. 17 IN AAAA ::1") default: return nil } @@ -406,7 +107,7 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { return resolveAndCache(fqdn, qtype, securityLevel) } - if rrCache.Expires <= time.Now().Unix() { + if rrCache.TTL <= time.Now().Unix() { rrCache.requestingNew = true go resolveAndCache(fqdn, qtype, securityLevel) } @@ -420,17 +121,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { return rrCache } -func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { +func resolveAndCache(fqdn string, qtype dns.Type, securityLevel uint8) (rrCache *RRCache) { // log.Tracef("intel: resolving %s%s", fqdn, qtype.String()) - rrCache, ok := checkDomainFronting(fqdn, qtype, securityLevel) - if ok { - if rrCache == nil { - return nil - } - return rrCache - } - // dedup requests dupKey := fmt.Sprintf("%s%s", fqdn, qtype.String()) dupReqLock.Lock() @@ -469,29 +162,29 @@ func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { // persist to database rrCache.Clean(600) - rrCache.CreateWithType(fqdn, qtype) + rrCache.Save() return rrCache } -func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { +func intelligentResolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache { // TODO: handle being offline // TODO: handle multiple network connections - if config.Changed() { - log.Info("intel: config changed, reloading resolvers") - loadResolvers(false) - } else if env.NetworkChanged() { - log.Info("intel: network changed, reloading resolvers") - loadResolvers(true) - } - config.RLock() - defer config.RUnlock() + // TODO: handle these in a separate goroutine + // if config.Changed() { + // log.Info("intel: config changed, reloading resolvers") + // loadResolvers(false) + // } else if env.NetworkChanged() { + // log.Info("intel: network changed, reloading resolvers") + // loadResolvers(true) + // } + resolversLock.RLock() defer resolversLock.RUnlock() - lastFailBoundary := time.Now().Unix() - config.DNSServerRetryRate + lastFailBoundary := time.Now().Unix() - nameserverRetryRate() preDottedFqdn := "." + fqdn // resolve: @@ -510,11 +203,14 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach } } // check config - if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) { + if doNotUseMulticastDNS(securityLevel) { return nil } // try mdns - rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel) + rrCache, err := queryMulticastDNS(fqdn, qtype) + if err != nil { + log.Errorf("intel: failed to query mdns: %s", err) + } return rrCache } @@ -533,15 +229,18 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach switch { case strings.HasSuffix(preDottedFqdn, ".local."): // check config - if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) { + if doNotUseMulticastDNS(securityLevel) { return nil } // try mdns - rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel) + rrCache, err := queryMulticastDNS(fqdn, qtype) + if err != nil { + log.Errorf("intel: failed to query mdns: %s", err) + } return rrCache case domainInScopes(preDottedFqdn, specialScopes): // check config - if config.DoNotForwardSpecialDomains.IsSetWithLevel(securityLevel) { + if doNotResolveSpecialDomains(securityLevel) { return nil } // try local resolvers @@ -568,15 +267,15 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach } -func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel int8) (*RRCache, bool) { +func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel uint8) (*RRCache, bool) { // skip if not allowed in current security level - if resolver.AllowedSecurityLevel < config.SecurityLevel() || resolver.AllowedSecurityLevel < securityLevel { - log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, config.SecurityLevel(), securityLevel) + if resolver.AllowedSecurityLevel < status.CurrentSecurityLevel() || resolver.AllowedSecurityLevel < securityLevel { + log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel) return nil, false } // skip if not security level denies assigned dns servers - if config.DoNotUseAssignedDNS.IsSetWithLevel(securityLevel) && resolver.Source == "dhcp" { - log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d (%d)", resolver, config.SecurityLevel(), securityLevel, int8(config.DoNotUseAssignedDNS)) + if doNotUseAssignedNameservers(securityLevel) && resolver.Source == "dhcp" { + log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel) return nil, false } // check if failed recently @@ -606,7 +305,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype } // resolve log.Tracef("intel: trying to resolve %s%s with %s", fqdn, qtype.String(), resolver.Server) - rrCache, err := resolver.Resolve(resolver, fqdn, qtype) + rrCache, err := query(resolver, fqdn, qtype) if err != nil { // check if failing is disabled if atomic.LoadInt64(resolver.LastFail) == -1 { @@ -625,7 +324,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype return rrCache, true } -func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { +func query(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { q := new(dns.Msg) q.SetQuestion(fqdn, uint16(qtype)) @@ -633,8 +332,7 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) var reply *dns.Msg var err error for i := 0; i < 5; i++ { - client := new(dns.Client) - reply, _, err = client.Exchange(q, resolver.ServerAddress) + reply, _, err = resolver.clientManager.getDNSClient().Exchange(q, resolver.ServerAddress) if err != nil { // TODO: handle special cases @@ -655,6 +353,8 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) } new := &RRCache{ + Domain: fqdn, + Question: qtype, Answer: reply.Answer, Ns: reply.Ns, Extra: reply.Extra, @@ -663,85 +363,3 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) // TODO: check if reply.Answer is valid return new, nil } - -type DnsOverHttpsReply struct { - Status uint32 - Truncated bool `json:"TC"` - Answer []DohRR - Additional []DohRR -} - -type DohRR struct { - Name string `json:"name"` - Qtype uint16 `json:"type"` - TTL uint32 `json:"TTL"` - Data string `json:"data"` -} - -func queryDNSoverHTTPS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { - - // API documentation: https://developers.google.com/speed/public-dns/docs/dns-over-https - - payload := url.Values{} - payload.Add("name", fqdn) - payload.Add("type", strconv.Itoa(int(qtype))) - payload.Add("edns_client_subnet", "0.0.0.0/0") - // TODO: add random - only use upper- and lower-case letters, digits, hyphen, period, underscore and tilde - // payload.Add("random_padding", "") - - resp, err := resolver.HTTPClient.Get(fmt.Sprintf("https://%s/resolve?%s", resolver.ServerAddress, payload.Encode())) - if err != nil { - return nil, fmt.Errorf("resolving %s%s failed: http error: %s", fqdn, qtype.String(), err) - // TODO: handle special cases - // 1. connect: network is unreachable - // intel: resolver DoH|dns.google.com:443|df:www.google.com failed (resolving discovery-v4-4.syncthing.net.A failed: http error: Get https://dns.google.com:443/resolve?edns_client_subnet=0.0.0.0%2F0&name=discovery-v4-4.syncthing.net.&type=1: dial tcp [2a00:1450:4001:819::2004]:443: connect: network is unreachable), moving to next - // 2. timeout - } - if resp.StatusCode != 200 { - return nil, fmt.Errorf("resolving %s%s failed: request was unsuccessful, got code %d", fqdn, qtype.String(), resp.StatusCode) - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("resolving %s%s failed: error reading response body: %s", fqdn, qtype.String(), err) - } - - var reply DnsOverHttpsReply - err = json.Unmarshal(body, &reply) - if err != nil { - return nil, fmt.Errorf("resolving %s%s failed: error parsing response body: %s", fqdn, qtype.String(), err) - } - - if reply.Status != 0 { - // this happens if there is a server error (e.g. DNSSEC fail), ignore for now - // TODO: do something more intelligent - } - - new := new(RRCache) - - // TODO: handle TXT records - - for _, entry := range reply.Answer { - rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)) - if err != nil { - log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data) - continue - } - new.Answer = append(new.Answer, rr) - } - - for _, entry := range reply.Additional { - rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)) - if err != nil { - log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data) - continue - } - new.Extra = append(new.Extra, rr) - } - - return new, nil -} - -// TODO: implement T-DNS: DNS over TCP/TLS -// server list: https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers diff --git a/intel/resolver.go b/intel/resolver.go new file mode 100644 index 00000000..ae210c93 --- /dev/null +++ b/intel/resolver.go @@ -0,0 +1,292 @@ +package intel + +import ( + "errors" + "fmt" + "net" + "sort" + "strconv" + "strings" + "sync" + + "github.com/miekg/dns" + "github.com/tevino/abool" + + "github.com/Safing/portbase/log" + + "github.com/Safing/portmaster/network/environment" + "github.com/Safing/portmaster/network/netutils" + "github.com/Safing/portmaster/status" +) + +// Resolver holds information about an active resolver. +type Resolver struct { + // static + Server string + ServerType string + ServerAddress string + ServerIP net.IP + ServerPort uint16 + VerifyDomain string + Source string + clientManager *clientManager + + Search *[]string + AllowedSecurityLevel uint8 + SkipFqdnBeforeInit string + + // atomic + Initialized *abool.AtomicBool + InitLock sync.Mutex + LastFail *int64 + Expires *int64 + + // must be locked + LockReason sync.Mutex + FailReason string + + // TODO: add: + // Expiration (for server got from DHCP / ICMPv6) + // bootstrapping (first query is already sent, wait for it to either succeed or fail - think about http bootstrapping here!) + // expanded server info: type, server address, server port, options - so we do not have to parse this every time! +} + +func (r *Resolver) String() string { + return r.Server +} + +// Scope defines a domain scope and which resolvers can resolve it. +type Scope struct { + Domain string + Resolvers []*Resolver +} + +var ( + globalResolvers []*Resolver // all resolvers + localResolvers []*Resolver // all resolvers that are in site-local or link-local IP ranges + localScopes []*Scope // list of scopes with a list of local resolvers that can resolve the scope + resolversLock sync.RWMutex + + env = environment.NewInterface() + + dupReqMap = make(map[string]*sync.Mutex) + dupReqLock sync.Mutex +) + +func indexOfResolver(server string, list []*Resolver) int { + for k, v := range list { + if v.Server == server { + return k + } + } + return -1 +} + +func indexOfScope(domain string, list []*Scope) int { + for k, v := range list { + if v.Domain == domain { + return k + } + } + return -1 +} + +func parseAddress(server string) (net.IP, uint16, error) { + delimiter := strings.LastIndex(server, ":") + if delimiter < 0 { + return nil, 0, errors.New("port missing") + } + ip := net.ParseIP(strings.Trim(server[:delimiter], "[]")) + if ip == nil { + return nil, 0, errors.New("invalid IP address") + } + port, err := strconv.Atoi(server[delimiter+1:]) + if err != nil || port < 1 || port > 65536 { + return nil, 0, errors.New("invalid port") + } + return ip, uint16(port), nil +} + +func urlFormatAddress(ip net.IP, port uint16) string { + var address string + if ipv4 := ip.To4(); ipv4 != nil { + address = fmt.Sprintf("%s:%d", ipv4.String(), port) + } else { + address = fmt.Sprintf("[%s]:%d", ip.String(), port) + } + return address +} + +func loadResolvers(resetResolvers bool) { + // TODO: what happens when a lot of processes want to reload at once? we do not need to run this multiple times in a short time frame. + resolversLock.Lock() + defer resolversLock.Unlock() + + var newResolvers []*Resolver + +configuredServersLoop: + for _, server := range configuredNameServers() { + key := indexOfResolver(server, newResolvers) + if key >= 0 { + continue configuredServersLoop + } + key = indexOfResolver(server, globalResolvers) + if resetResolvers || key == -1 { + + parts := strings.Split(server, "|") + if len(parts) < 2 { + log.Warningf("intel: nameserver format invalid: %s", server) + continue configuredServersLoop + } + + ip, port, err := parseAddress(parts[1]) + if err != nil && strings.ToLower(parts[0]) != "https" { + log.Warningf("intel: nameserver (%s) address invalid: %s", server, err) + continue configuredServersLoop + } + + var lastFail int64 + new := &Resolver{ + Server: server, + ServerType: parts[0], + ServerAddress: parts[1], + ServerIP: ip, + ServerPort: port, + LastFail: &lastFail, + Source: "config", + Initialized: abool.NewBool(false), + } + + switch strings.ToLower(parts[0]) { + case "dns": + new.clientManager = newDNSClientManager(new) + case "tcp": + new.clientManager = newTCPClientManager(new) + case "tls": + new.AllowedSecurityLevel = status.SecurityLevelFortress + if len(parts) < 3 { + log.Warningf("intel: nameserver missing verification domain as third parameter: %s", server) + continue configuredServersLoop + } + new.VerifyDomain = parts[2] + new.clientManager = newTLSClientManager(new) + case "https": + new.AllowedSecurityLevel = status.SecurityLevelFortress + new.SkipFqdnBeforeInit = dns.Fqdn(strings.Split(parts[1], ":")[0]) + if len(parts) > 2 { + new.VerifyDomain = parts[2] + } + new.clientManager = newHTTPSClientManager(new) + default: + log.Warningf("intel: nameserver (%s) type invalid: %s", server, parts[0]) + continue configuredServersLoop + } + newResolvers = append(newResolvers, new) + } else { + newResolvers = append(newResolvers, globalResolvers[key]) + } + } + + // add local resolvers + assignedNameservers := environment.Nameservers() +assignedServersLoop: + for _, nameserver := range assignedNameservers { + server := fmt.Sprintf("dns|%s", urlFormatAddress(nameserver.IP, 53)) + key := indexOfResolver(server, newResolvers) + if key >= 0 { + continue assignedServersLoop + } + key = indexOfResolver(server, globalResolvers) + if resetResolvers || key == -1 { + + var lastFail int64 + new := &Resolver{ + Server: server, + ServerType: "dns", + ServerAddress: urlFormatAddress(nameserver.IP, 53), + ServerIP: nameserver.IP, + ServerPort: 53, + LastFail: &lastFail, + Source: "dhcp", + Initialized: abool.NewBool(false), + AllowedSecurityLevel: status.SecurityLevelSecure, + } + new.clientManager = newDNSClientManager(new) + + if netutils.IPIsLocal(nameserver.IP) && len(nameserver.Search) > 0 { + // only allow searches for local resolvers + var newSearch []string + for _, value := range nameserver.Search { + newSearch = append(newSearch, fmt.Sprintf(".%s.", strings.Trim(value, "."))) + } + new.Search = &newSearch + } + newResolvers = append(newResolvers, new) + } else { + newResolvers = append(newResolvers, globalResolvers[key]) + } + } + + // save resolvers + globalResolvers = newResolvers + if len(globalResolvers) == 0 { + log.Criticalf("intel: no (valid) dns servers found in configuration and system") + } + + // make list with local resolvers + localResolvers = make([]*Resolver, 0) + for _, resolver := range globalResolvers { + if resolver.ServerIP != nil && netutils.IPIsLocal(resolver.ServerIP) { + localResolvers = append(localResolvers, resolver) + } + } + + // add resolvers to every scope the cover + localScopes = make([]*Scope, 0) + for _, resolver := range globalResolvers { + + if resolver.Search != nil { + // add resolver to custom searches + for _, search := range *resolver.Search { + if search == "." { + continue + } + key := indexOfScope(search, localScopes) + if key == -1 { + localScopes = append(localScopes, &Scope{ + Domain: search, + Resolvers: []*Resolver{resolver}, + }) + } else { + localScopes[key].Resolvers = append(localScopes[key].Resolvers, resolver) + } + } + + } + } + + // sort scopes by length + sort.Slice(localScopes, + func(i, j int) bool { + return len(localScopes[i].Domain) > len(localScopes[j].Domain) + }, + ) + + log.Trace("intel: loaded global resolvers:") + for _, resolver := range globalResolvers { + log.Tracef("intel: %s", resolver.Server) + } + log.Trace("intel: loaded local resolvers:") + for _, resolver := range localResolvers { + log.Tracef("intel: %s", resolver.Server) + } + log.Trace("intel: loaded scopes:") + for _, scope := range localScopes { + var scopeServers []string + for _, resolver := range scope.Resolvers { + scopeServers = append(scopeServers, resolver.Server) + } + log.Tracef("intel: %s: %s", scope.Domain, strings.Join(scopeServers, ", ")) + } + +} diff --git a/intel/rrcache.go b/intel/rrcache.go new file mode 100644 index 00000000..985053fc --- /dev/null +++ b/intel/rrcache.go @@ -0,0 +1,164 @@ +// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file. + +package intel + +import ( + "net" + "time" + + "github.com/miekg/dns" +) + +// RRCache is used to cache DNS data +type RRCache struct { + Domain string + Question dns.Type + + Answer []dns.RR + Ns []dns.RR + Extra []dns.RR + TTL int64 + + updated int64 + servedFromCache bool + requestingNew bool +} + +// Clean sets all TTLs to 17 and sets cache expiry with specified minimum. +func (m *RRCache) Clean(minExpires uint32) { + var lowestTTL uint32 = 0xFFFFFFFF + var header *dns.RR_Header + + // set TTLs to 17 + // TODO: double append? is there something more elegant? + for _, rr := range append(m.Answer, append(m.Ns, m.Extra...)...) { + header = rr.Header() + if lowestTTL > header.Ttl { + lowestTTL = header.Ttl + } + header.Ttl = 17 + } + + // TTL must be at least minExpires + if lowestTTL < minExpires { + lowestTTL = minExpires + } + + m.TTL = time.Now().Unix() + int64(lowestTTL) +} + +// ExportAllARecords return of a list of all A and AAAA IP addresses. +func (m *RRCache) ExportAllARecords() (ips []net.IP) { + for _, rr := range m.Answer { + if rr.Header().Class != dns.ClassINET { + continue + } + + switch rr.Header().Rrtype { + case dns.TypeA: + aRecord, ok := rr.(*dns.A) + if ok { + ips = append(ips, aRecord.A) + } + case dns.TypeAAAA: + aaaaRecord, ok := rr.(*dns.AAAA) + if ok { + ips = append(ips, aaaaRecord.AAAA) + } + } + } + return +} + +// ToNameRecord converts the RRCache to a NameRecord for cleaner persistence. +func (m *RRCache) ToNameRecord() *NameRecord { + new := &NameRecord{ + Domain: m.Domain, + Question: m.Question.String(), + TTL: m.TTL, + } + + // stringify RR entries + for _, entry := range m.Answer { + new.Answer = append(new.Answer, entry.String()) + } + for _, entry := range m.Ns { + new.Ns = append(new.Ns, entry.String()) + } + for _, entry := range m.Extra { + new.Extra = append(new.Extra, entry.String()) + } + + return new +} + +// Save saves the RRCache to the database as a NameRecord. +func (m *RRCache) Save() error { + return m.ToNameRecord().Save() +} + +// GetRRCache tries to load the corresponding NameRecord from the database and convert it. +func GetRRCache(domain string, question dns.Type) (*RRCache, error) { + var m RRCache + rr := &RRCache{ + Domain: domain, + Question: question, + } + + nameRecord, err := GetNameRecord(domain, question.String()) + if err != nil { + return nil, err + } + + rr.TTL = nameRecord.TTL + for _, entry := range nameRecord.Answer { + rr, err := dns.NewRR(entry) + if err == nil { + m.Answer = append(m.Answer, rr) + } + } + for _, entry := range nameRecord.Ns { + rr, err := dns.NewRR(entry) + if err == nil { + m.Ns = append(m.Ns, rr) + } + } + for _, entry := range nameRecord.Extra { + rr, err := dns.NewRR(entry) + if err == nil { + m.Extra = append(m.Extra, rr) + } + } + + m.servedFromCache = true + return &m, nil +} + +// ServedFromCache marks the RRCache as served from cache. +func (m *RRCache) ServedFromCache() bool { + return m.servedFromCache +} + +// RequestingNew informs that it has expired and new RRs are being fetched. +func (m *RRCache) RequestingNew() bool { + return m.requestingNew +} + +// Flags formats ServedFromCache and RequestingNew to a condensed, flag-like format. +func (m *RRCache) Flags() string { + switch { + case m.servedFromCache && m.requestingNew: + return " [CR]" + case m.servedFromCache: + return " [C]" + case m.requestingNew: + return " [R]" // should never enter this state, but let's leave it here, just in case + default: + return "" + } +} + +// IsNXDomain returnes whether the result is nxdomain. +func (m *RRCache) IsNXDomain() bool { + return len(m.Answer) == 0 +} diff --git a/intel/special.go b/intel/special.go index 476e457c..c979599b 100644 --- a/intel/special.go +++ b/intel/special.go @@ -5,7 +5,7 @@ package intel import "strings" var ( - localReverseScopes = &[]string{ + localReverseScopes = []string{ ".10.in-addr.arpa.", ".16.172.in-addr.arpa.", ".17.172.in-addr.arpa.", @@ -31,7 +31,8 @@ var ( ".b.e.f.ip6.arpa.", } - specialScopes = &[]string{ + // RFC6761, RFC7686 + specialScopes = []string{ ".example.", ".example.com.", ".example.net.", @@ -42,8 +43,8 @@ var ( } ) -func domainInScopes(fqdn string, list *[]string) bool { - for _, scope := range *list { +func domainInScopes(fqdn string, list []string) bool { + for _, scope := range list { if strings.HasSuffix(fqdn, scope) { return true }