github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Start intel package adjustments

Browse Source
This commit is contained in:
Daniel
2018-10-22 17:03:23 +02:00
parent b1cd19a8e8
commit bf55c1232d
14 changed files with 912 additions and 842 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
intel/clients.go Normal file
View File

@@ -0,0 +1,94 @@
package intel
import (
"crypto/tls"
"sync"
"time"
"github.com/miekg/dns"
)
type clientManager struct {
dnsClient *dns.Client
factory func() *dns.Client
lock sync.Mutex
refreshAfter time.Time
ttl time.Duration // force refresh of connection to reduce traceability
}
// ref: https://godoc.org/github.com/miekg/dns#Client
func newDNSClientManager(resolver *Resolver) *clientManager {
return &clientManager{
ttl: -1 * time.Minute,
factory: func() *dns.Client {
return &dns.Client{
Timeout: 5 * time.Second,
}
},
}
}
func newTCPClientManager(resolver *Resolver) *clientManager {
return &clientManager{
ttl: -15 * time.Minute,
factory: func() *dns.Client {
return &dns.Client{
Net: "tcp",
Timeout: 5 * time.Second,
}
},
}
}
func newTLSClientManager(resolver *Resolver) *clientManager {
return &clientManager{
ttl: -15 * time.Minute,
factory: func() *dns.Client {
return &dns.Client{
Net: "tcp-tls",
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: resolver.VerifyDomain,
// TODO: use custom random
// Rand: io.Reader,
},
Timeout: 5 * time.Second,
}
},
}
}
func newHTTPSClientManager(resolver *Resolver) *clientManager {
return &clientManager{
ttl: -15 * time.Minute,
factory: func() *dns.Client {
new := &dns.Client{
Net: "https",
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
// TODO: use custom random
// Rand: io.Reader,
},
Timeout: 5 * time.Second,
}
if resolver.VerifyDomain != "" {
new.TLSConfig.ServerName = resolver.VerifyDomain
}
return new
},
}
}
func (cm *clientManager) getDNSClient() *dns.Client {
cm.lock.Lock()
defer cm.lock.Unlock()
if cm.dnsClient == nil || time.Now().After(cm.refreshAfter) {
cm.dnsClient = cm.factory()
cm.refreshAfter = time.Now().Add(cm.ttl)
}
return cm.dnsClient
}

83
intel/config.go Normal file
View File

@@ -0,0 +1,83 @@
package intel
import (
"github.com/Safing/portbase/config"
"github.com/Safing/portmaster/status"
)
var (
configuredNameServers config.StringArrayOption
defaultNameServers = []string{
"tls|1.1.1.1:853|cloudflare-dns.com", // Cloudflare
"tls|1.0.0.1:853|cloudflare-dns.com", // Cloudflare
"tls|9.9.9.9:853|dns.quad9.net", // Quad9
// "https|cloudflare-dns.com/dns-query", // HTTPS still experimental
"dns|1.1.1.1:53", // Cloudflare
"dns|1.0.0.1:53", // Cloudflare
"dns|9.9.9.9:53", // Quad9
}
nameserverRetryRate config.IntOption
doNotUseMulticastDNS status.SecurityLevelOption
doNotUseAssignedNameservers status.SecurityLevelOption
doNotResolveSpecialDomains status.SecurityLevelOption
)
func init() {
config.Register(&config.Option{
Name: "Nameservers (DNS)",
Key: "intel/nameservers",
Description: "Nameserver to use for resolving DNS requests.",
ExpertiseLevel: config.ExpertiseLevelExpert,
OptType: config.OptTypeStringArray,
DefaultValue: defaultNameServers,
ValidationRegex: "^(dns|tcp|tls|https)$",
})
configuredNameServers = config.Concurrent.GetAsStringArray("intel/nameservers", defaultNameServers)
config.Register(&config.Option{
Name: "Nameserver Retry Rate",
Key: "intel/nameserverRetryRate",
Description: "Rate at which to retry failed nameservers, in seconds.",
ExpertiseLevel: config.ExpertiseLevelExpert,
OptType: config.OptTypeInt,
DefaultValue: 600,
})
nameserverRetryRate = config.Concurrent.GetAsInt("intel/nameserverRetryRate", 0)
config.Register(&config.Option{
Name: "Do not use Multicast DNS",
Key: "intel/doNotUseMulticastDNS",
Description: "",
ExpertiseLevel: config.ExpertiseLevelExpert,
OptType: config.OptTypeInt,
ExternalOptType: "security level",
DefaultValue: 3,
ValidationRegex: "^(1|2|3)$",
})
doNotUseMulticastDNS = status.ConfigIsActiveConcurrent("intel/doNotUseMulticastDNS")
config.Register(&config.Option{
Name: "Do not use assigned Nameservers",
Key: "intel/doNotUseAssignedNameservers",
Description: "that were acquired by the network (dhcp) or system",
ExpertiseLevel: config.ExpertiseLevelExpert,
OptType: config.OptTypeInt,
ExternalOptType: "security level",
DefaultValue: 3,
ValidationRegex: "^(1|2|3)$",
})
doNotUseAssignedNameservers = status.ConfigIsActiveConcurrent("intel/doNotUseAssignedNameservers")
config.Register(&config.Option{
Name: "Do not resolve special domains",
Key: "intel/doNotResolveSpecialDomains",
Description: "Do not resolve special (top level) domains: example, example.com, example.net, example.org, invalid, test, onion. (RFC6761, RFC7686)",
ExpertiseLevel: config.ExpertiseLevelExpert,
OptType: config.OptTypeInt,
ExternalOptType: "security level",
DefaultValue: 3,
ValidationRegex: "^(1|2|3)$",
})
doNotResolveSpecialDomains = status.ConfigIsActiveConcurrent("intel/doNotResolveSpecialDomains")
}

62
intel/data.go
View File

@@ -1,62 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package intel
import (
"github.com/Safing/safing-core/database"
datastore "github.com/ipfs/go-datastore"
)
// EntityClassification holds classification information about an internet entity.
type EntityClassification struct {
lists []byte
}
// Intel holds intelligence data for a domain.
type Intel struct {
database.Base
Domain string
DomainOwner string
CertOwner string
Classification *EntityClassification
}
var intelModel *Intel // only use this as parameter for database.EnsureModel-like functions
func init() {
database.RegisterModel(intelModel, func() database.Model { return new(Intel) })
}
// Create saves the Intel with the provided name in the default namespace.
func (m *Intel) Create(name string) error {
return m.CreateObject(&database.IntelCache, name, m)
}
// CreateInNamespace saves the Intel with the provided name in the provided namespace.
func (m *Intel) CreateInNamespace(namespace *datastore.Key, name string) error {
return m.CreateObject(namespace, name, m)
}
// Save saves the Intel.
func (m *Intel) Save() error {
return m.SaveObject(m)
}
// getIntel fetches the Intel with the provided name in the default namespace.
func getIntel(name string) (*Intel, error) {
return getIntelFromNamespace(&database.IntelCache, name)
}
// getIntelFromNamespace fetches the Intel with the provided name in the provided namespace.
func getIntelFromNamespace(namespace *datastore.Key, name string) (*Intel, error) {
object, err := database.GetAndEnsureModel(namespace, name, intelModel)
if err != nil {
return nil, err
}
model, ok := object.(*Intel)
if !ok {
return nil, database.NewMismatchError(object, intelModel)
}
return model, nil
}

218
intel/dns.go
View File

@@ -1,218 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package intel
import (
"fmt"
"net"
"time"
"github.com/Safing/safing-core/database"
datastore "github.com/ipfs/go-datastore"
"github.com/miekg/dns"
)
// RRCache is used to cache DNS data
type RRCache struct {
Answer []dns.RR
Ns []dns.RR
Extra []dns.RR
Expires int64
Modified int64
servedFromCache bool
requestingNew bool
}
func (m *RRCache) Clean(minExpires uint32) {
var lowestTTL uint32 = 0xFFFFFFFF
var header *dns.RR_Header
// set TTLs to 17
// TODO: double append? is there something more elegant?
for _, rr := range append(m.Answer, append(m.Ns, m.Extra...)...) {
header = rr.Header()
if lowestTTL > header.Ttl {
lowestTTL = header.Ttl
}
header.Ttl = 17
}
// TTL must be at least minExpires
if lowestTTL < minExpires {
lowestTTL = minExpires
}
m.Expires = time.Now().Unix() + int64(lowestTTL)
m.Modified = time.Now().Unix()
}
func (m *RRCache) ExportAllARecords() (ips []net.IP) {
for _, rr := range m.Answer {
if rr.Header().Class == dns.ClassINET && rr.Header().Rrtype == dns.TypeA {
aRecord, ok := rr.(*dns.A)
if ok {
ips = append(ips, aRecord.A)
}
} else if rr.Header().Class == dns.ClassINET && rr.Header().Rrtype == dns.TypeAAAA {
aRecord, ok := rr.(*dns.AAAA)
if ok {
ips = append(ips, aRecord.AAAA)
}
}
}
return
}
func (m *RRCache) ToRRSave() *RRSave {
var s RRSave
s.Expires = m.Expires
s.Modified = m.Modified
for _, entry := range m.Answer {
s.Answer = append(s.Answer, entry.String())
}
for _, entry := range m.Ns {
s.Ns = append(s.Ns, entry.String())
}
for _, entry := range m.Extra {
s.Extra = append(s.Extra, entry.String())
}
return &s
}
func (m *RRCache) Create(name string) error {
s := m.ToRRSave()
return s.CreateObject(&database.DNSCache, name, s)
}
func (m *RRCache) CreateWithType(name string, qtype dns.Type) error {
s := m.ToRRSave()
return s.Create(fmt.Sprintf("%s%s", name, qtype.String()))
}
func (m *RRCache) Save() error {
s := m.ToRRSave()
return s.SaveObject(s)
}
func GetRRCache(domain string, qtype dns.Type) (*RRCache, error) {
return GetRRCacheFromNamespace(&database.DNSCache, domain, qtype)
}
func GetRRCacheFromNamespace(namespace *datastore.Key, domain string, qtype dns.Type) (*RRCache, error) {
var m RRCache
rrSave, err := GetRRSaveFromNamespace(namespace, domain, qtype)
if err != nil {
return nil, err
}
m.Expires = rrSave.Expires
m.Modified = rrSave.Modified
for _, entry := range rrSave.Answer {
rr, err := dns.NewRR(entry)
if err == nil {
m.Answer = append(m.Answer, rr)
}
}
for _, entry := range rrSave.Ns {
rr, err := dns.NewRR(entry)
if err == nil {
m.Ns = append(m.Ns, rr)
}
}
for _, entry := range rrSave.Extra {
rr, err := dns.NewRR(entry)
if err == nil {
m.Extra = append(m.Extra, rr)
}
}
m.servedFromCache = true
return &m, nil
}
// ServedFromCache marks the RRCache as served from cache.
func (m *RRCache) ServedFromCache() bool {
return m.servedFromCache
}
// RequestingNew informs that it has expired and new RRs are being fetched.
func (m *RRCache) RequestingNew() bool {
return m.requestingNew
}
// Flags formats ServedFromCache and RequestingNew to a condensed, flag-like format.
func (m *RRCache) Flags() string {
switch {
case m.servedFromCache && m.requestingNew:
return " [CR]"
case m.servedFromCache:
return " [C]"
case m.requestingNew:
return " [R]" // theoretically impossible, but let's leave it here, just in case
default:
return ""
}
}
// IsNXDomain returnes whether the result is nxdomain.
func (m *RRCache) IsNXDomain() bool {
return len(m.Answer) == 0
}
// RRSave is helper struct to RRCache to better save data to the database.
type RRSave struct {
database.Base
Answer []string
Ns []string
Extra []string
Expires int64
Modified int64
}
var rrSaveModel *RRSave // only use this as parameter for database.EnsureModel-like functions
func init() {
database.RegisterModel(rrSaveModel, func() database.Model { return new(RRSave) })
}
// Create saves RRSave with the provided name in the default namespace.
func (m *RRSave) Create(name string) error {
return m.CreateObject(&database.DNSCache, name, m)
}
// CreateWithType saves RRSave with the provided name and type in the default namespace.
func (m *RRSave) CreateWithType(name string, qtype dns.Type) error {
return m.Create(fmt.Sprintf("%s%s", name, qtype.String()))
}
// CreateInNamespace saves RRSave with the provided name in the provided namespace.
func (m *RRSave) CreateInNamespace(namespace *datastore.Key, name string) error {
return m.CreateObject(namespace, name, m)
}
// Save saves RRSave.
func (m *RRSave) Save() error {
return m.SaveObject(m)
}
// GetRRSave fetches RRSave with the provided name in the default namespace.
func GetRRSave(name string, qtype dns.Type) (*RRSave, error) {
return GetRRSaveFromNamespace(&database.DNSCache, name, qtype)
}
// GetRRSaveFromNamespace fetches RRSave with the provided name in the provided namespace.
func GetRRSaveFromNamespace(namespace *datastore.Key, name string, qtype dns.Type) (*RRSave, error) {
object, err := database.GetAndEnsureModel(namespace, fmt.Sprintf("%s%s", name, qtype.String()), rrSaveModel)
if err != nil {
return nil, err
}
model, ok := object.(*RRSave)
if !ok {
return nil, database.NewMismatchError(object, rrSaveModel)
}
return model, nil
}

48
intel/domainfronting.go
View File

@@ -1,48 +0,0 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package intel
import (
"github.com/Safing/safing-core/log"
"sync"
"github.com/miekg/dns"
)
var (
dfMap = make(map[string]string)
dfMapLock sync.RWMutex
)
func checkDomainFronting(hidden string, qtype dns.Type, securityLevel int8) (*RRCache, bool) {
dfMapLock.RLock()
front, ok := dfMap[hidden]
dfMapLock.RUnlock()
if !ok {
return nil, false
}
log.Tracef("intel: applying domain fronting %s -> %s", hidden, front)
// get domain name
rrCache := resolveAndCache(front, qtype, securityLevel)
if rrCache == nil {
return nil, true
}
// replace domain name
var header *dns.RR_Header
for _, rr := range rrCache.Answer {
header = rr.Header()
if header.Name == front {
header.Name = hidden
}
}
// save under front
rrCache.CreateWithType(hidden, qtype)
return rrCache, true
}
func addDomainFronting(hidden string, front string) {
dfMapLock.Lock()
dfMap[hidden] = front
dfMapLock.Unlock()
return
}

74
intel/intel.go
View File

@@ -3,44 +3,66 @@
package intel package intel
import ( import (
"github.com/Safing/safing-core/database" "fmt"
"github.com/Safing/safing-core/modules" "sync"
"github.com/miekg/dns" "github.com/Safing/portbase/database"
"github.com/Safing/portbase/database/record"
) )
var ( var (
intelModule *modules.Module intelDatabase = database.NewInterface(&database.Options{
AlwaysSetRelativateExpiry: 2592000, // 30 days
})
) )
func init() { // Intel holds intelligence data for a domain.
intelModule = modules.Register("Intel", 128) type Intel struct {
go Start() record.Base
sync.Mutex
Domain string
} }
// GetIntel returns an Intel object of the given domain. The returned Intel object MUST not be modified. func makeIntelKey(domain string) string {
func GetIntel(domain string) *Intel { return fmt.Sprintf("intel:Intel/%s", domain)
fqdn := dns.Fqdn(domain) }
intel, err := getIntel(fqdn)
// GetIntelFromDB gets an Intel record from the database.
func GetIntelFromDB(domain string) (*Intel, error) {
key := makeIntelKey(domain)
r, err := intelDatabase.Get(key)
if err != nil { if err != nil {
if err == database.ErrNotFound { return nil, err
intel = &Intel{Domain: fqdn}
intel.Create(fqdn)
} else {
return nil
}
} }
return intel
// unwrap
if r.IsWrapped() {
// only allocate a new struct, if we need it
new := &Intel{}
err = record.Unwrap(r, new)
if err != nil {
return nil, err
}
return new, nil
}
// or adjust type
new, ok := r.(*Intel)
if !ok {
return nil, fmt.Errorf("record not of type *Intel, but %T", r)
}
return new, nil
} }
func GetIntelAndRRs(domain string, qtype dns.Type, securityLevel int8) (intel *Intel, rrs *RRCache) { // Save saves the Intel record to the database.
intel = GetIntel(domain) func (intel *Intel) Save() error {
rrs = Resolve(domain, qtype, securityLevel) intel.SetKey(makeIntelKey(intel.Domain))
return return intelDatabase.PutNew(intel)
} }
func Start() { // GetIntel fetches intelligence data for the given domain.
// mocking until intel has its own goroutines func GetIntel(domain string) (*Intel, error) {
defer intelModule.StopComplete() return &Intel{Domain: domain}, nil
<-intelModule.Stop
} }

82
intel/ipinfo.go
View File

@@ -1,61 +1,69 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package intel package intel
import ( import (
"fmt"
"net"
"strings" "strings"
"sync"
"github.com/Safing/safing-core/database" "github.com/Safing/portbase/database"
"github.com/Safing/portbase/database/record"
)
datastore "github.com/ipfs/go-datastore" var (
ipInfoDatabase = database.NewInterface(&database.Options{
AlwaysSetRelativateExpiry: 86400, // 24 hours
})
) )
// IPInfo represents various information about an IP. // IPInfo represents various information about an IP.
type IPInfo struct { type IPInfo struct {
database.Base record.Base
sync.Mutex
IP net.IP
Domains []string Domains []string
} }
var ipInfoModel *IPInfo // only use this as parameter for database.EnsureModel-like functions func makeIPInfoKey(ip net.IP) string {
return fmt.Sprintf("intel:IPInfo/%s", ip.String())
func init() {
database.RegisterModel(ipInfoModel, func() database.Model { return new(IPInfo) })
} }
// Create saves the IPInfo with the provided name in the default namespace. // GetIPInfo gets an IPInfo record from the database.
func (m *IPInfo) Create(name string) error { func GetIPInfo(ip net.IP) (*IPInfo, error) {
return m.CreateObject(&database.IPInfoCache, name, m) key := makeIPInfoKey(ip)
}
// CreateInNamespace saves the IPInfo with the provided name in the provided namespace. r, err := ipInfoDatabase.Get(key)
func (m *IPInfo) CreateInNamespace(namespace *datastore.Key, name string) error {
return m.CreateObject(namespace, name, m)
}
// Save saves the IPInfo.
func (m *IPInfo) Save() error {
return m.SaveObject(m)
}
// GetIPInfo fetches the IPInfo with the provided name in the default namespace.
func GetIPInfo(name string) (*IPInfo, error) {
return GetIPInfoFromNamespace(&database.IPInfoCache, name)
}
// GetIPInfoFromNamespace fetches the IPInfo with the provided name in the provided namespace.
func GetIPInfoFromNamespace(namespace *datastore.Key, name string) (*IPInfo, error) {
object, err := database.GetAndEnsureModel(namespace, name, ipInfoModel)
if err != nil { if err != nil {
return nil, err return nil, err
} }
model, ok := object.(*IPInfo)
if !ok { // unwrap
return nil, database.NewMismatchError(object, ipInfoModel) if r.IsWrapped() {
// only allocate a new struct, if we need it
new := &IPInfo{}
err = record.Unwrap(r, new)
if err != nil {
return nil, err
}
return new, nil
} }
return model, nil
// or adjust type
new, ok := r.(*IPInfo)
if !ok {
return nil, fmt.Errorf("record not of type *IPInfo, but %T", r)
}
return new, nil
}
// Save saves the IPInfo record to the database.
func (ipi *IPInfo) Save() error {
ipi.SetKey(makeIPInfoKey(ipi.IP))
return ipInfoDatabase.PutNew(ipi)
} }
// FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or " // FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or "
func (m *IPInfo) FmtDomains() string { func (ipi *IPInfo) FmtDomains() string {
return strings.Join(m.Domains, " or ") return strings.Join(ipi.Domains, " or ")
} }

40
intel/main.go Normal file
View File

@@ -0,0 +1,40 @@
package intel
import (
"github.com/miekg/dns"
"github.com/Safing/portbase/database"
"github.com/Safing/portbase/log"
"github.com/Safing/portbase/modules"
)
func init() {
modules.Register("intel", nil, start, nil, "database")
}
func start() error {
_, err := database.Register(&database.Database{
Name: "intel",
Description: "Intelligence and DNS Data",
StorageType: "badger",
PrimaryAPI: "",
})
if err != nil {
return err
}
// load resolvers from config and environment
loadResolvers(false)
return nil
}
func GetIntelAndRRs(domain string, qtype dns.Type, securityLevel uint8) (intel *Intel, rrs *RRCache) {
intel, err := GetIntel(domain)
if err != nil {
log.Errorf("intel: failed to get intel: %s", err)
intel = nil
}
rrs = Resolve(domain, qtype, securityLevel)
return
}

52
intel/mdns.go
View File

@@ -6,12 +6,13 @@ import (
"errors" "errors"
"fmt" "fmt"
"net" "net"
"github.com/Safing/safing-core/log"
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/Safing/portbase/log"
) )
const ( const (
@@ -89,7 +90,7 @@ func listenToMDNS() {
var question *dns.Question var question *dns.Question
var saveFullRequest bool var saveFullRequest bool
scavengedRecords := make(map[string]*dns.RR) scavengedRecords := make(map[string]dns.RR)
var rrCache *RRCache var rrCache *RRCache
// save every received response // save every received response
@@ -114,7 +115,7 @@ func listenToMDNS() {
continue continue
} }
// continue if no question // get question, some servers do not reply with question
if len(message.Question) == 0 { if len(message.Question) == 0 {
questionsLock.Lock() questionsLock.Lock()
savedQ, ok := questions[message.MsgHdr.Id] savedQ, ok := questions[message.MsgHdr.Id]
@@ -138,8 +139,11 @@ func listenToMDNS() {
// get entry from database // get entry from database
if saveFullRequest { if saveFullRequest {
rrCache, err = GetRRCache(question.Name, dns.Type(question.Qtype)) rrCache, err = GetRRCache(question.Name, dns.Type(question.Qtype))
if err != nil || rrCache.Modified < time.Now().Add(-2*time.Second).Unix() || rrCache.Expires < time.Now().Unix() { if err != nil || rrCache.updated < time.Now().Add(-2*time.Second).Unix() || rrCache.TTL < time.Now().Unix() {
rrCache = &RRCache{} rrCache = &RRCache{
Domain: question.Name,
Question: dns.Type(question.Qtype),
}
} }
} }
@@ -155,12 +159,12 @@ func listenToMDNS() {
} }
switch entry.(type) { switch entry.(type) {
case *dns.A: case *dns.A:
scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = entry
case *dns.AAAA: case *dns.AAAA:
scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = entry
case *dns.PTR: case *dns.PTR:
if !strings.HasPrefix(entry.Header().Name, "_") { if !strings.HasPrefix(entry.Header().Name, "_") {
scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = entry
} }
} }
} }
@@ -177,17 +181,16 @@ func listenToMDNS() {
} }
switch entry.(type) { switch entry.(type) {
case *dns.A: case *dns.A:
scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%s_A", entry.Header().Name)] = entry
case *dns.AAAA: case *dns.AAAA:
scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%s_AAAA", entry.Header().Name)] = entry
case *dns.PTR: case *dns.PTR:
if !strings.HasPrefix(entry.Header().Name, "_") { if !strings.HasPrefix(entry.Header().Name, "_") {
scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%s_PTR", entry.Header().Name)] = entry
} }
} }
} }
} }
// TODO: scan Extra for A and AAAA records and save them seperately
for _, entry := range message.Extra { for _, entry := range message.Extra {
if strings.HasSuffix(entry.Header().Name, ".local.") || domainInScopes(entry.Header().Name, localReverseScopes) { if strings.HasSuffix(entry.Header().Name, ".local.") || domainInScopes(entry.Header().Name, localReverseScopes) {
if saveFullRequest { if saveFullRequest {
@@ -200,34 +203,35 @@ func listenToMDNS() {
} }
switch entry.(type) { switch entry.(type) {
case *dns.A: case *dns.A:
scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sA", entry.Header().Name)] = entry
case *dns.AAAA: case *dns.AAAA:
scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sAAAA", entry.Header().Name)] = entry
case *dns.PTR: case *dns.PTR:
if !strings.HasPrefix(entry.Header().Name, "_") { if !strings.HasPrefix(entry.Header().Name, "_") {
scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = &entry scavengedRecords[fmt.Sprintf("%sPTR", entry.Header().Name)] = entry
} }
} }
} }
} }
var questionID string
if saveFullRequest { if saveFullRequest {
rrCache.Clean(60) rrCache.Clean(60)
rrCache.CreateWithType(question.Name, dns.Type(question.Qtype)) rrCache.Save()
// log.Tracef("intel: mdns saved full reply to %s%s", question.Name, dns.Type(question.Qtype).String()) questionID = fmt.Sprintf("%s%s", question.Name, dns.Type(question.Qtype).String())
} }
for k, v := range scavengedRecords { for k, v := range scavengedRecords {
if saveFullRequest { if saveFullRequest && k == questionID {
if k == fmt.Sprintf("%s%s", question.Name, dns.Type(question.Qtype).String()) { continue
continue
}
} }
rrCache = &RRCache{ rrCache = &RRCache{
Answer: []dns.RR{*v}, Domain: v.Header().Name,
Question: dns.Type(v.Header().Class),
Answer: []dns.RR{v},
} }
rrCache.Clean(60) rrCache.Clean(60)
rrCache.Create(k) rrCache.Save()
// log.Tracef("intel: mdns scavenged %s", k) // log.Tracef("intel: mdns scavenged %s", k)
} }
@@ -261,7 +265,7 @@ func listenForDNSPackets(conn *net.UDPConn, messages chan *dns.Msg) {
} }
} }
func queryMulticastDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { func queryMulticastDNS(fqdn string, qtype dns.Type) (*RRCache, error) {
q := new(dns.Msg) q := new(dns.Msg)
q.SetQuestion(fqdn, uint16(qtype)) q.SetQuestion(fqdn, uint16(qtype))
// request unicast response // request unicast response

72
intel/namerecord.go Normal file
View File

@@ -0,0 +1,72 @@
package intel
import (
"errors"
"fmt"
"sync"
"github.com/Safing/portbase/database"
"github.com/Safing/portbase/database/record"
)
var (
recordDatabase = database.NewInterface(&database.Options{
AlwaysSetRelativateExpiry: 2592000, // 30 days
CacheSize: 100,
})
)
// NameRecord is helper struct to RRCache to better save data to the database.
type NameRecord struct {
record.Base
sync.Mutex
Domain string
Question string
Answer []string
Ns []string
Extra []string
TTL int64
}
func makeNameRecordKey(domain string, question string) string {
return fmt.Sprintf("intel:NameRecords/%s%s", domain, question)
}
// GetNameRecord gets a NameRecord from the database.
func GetNameRecord(domain string, question string) (*NameRecord, error) {
key := makeNameRecordKey(domain, question)
r, err := recordDatabase.Get(key)
if err != nil {
return nil, err
}
// unwrap
if r.IsWrapped() {
// only allocate a new struct, if we need it
new := &NameRecord{}
err = record.Unwrap(r, new)
if err != nil {
return nil, err
}
return new, nil
}
// or adjust type
new, ok := r.(*NameRecord)
if !ok {
return nil, fmt.Errorf("record not of type *NameRecord, but %T", r)
}
return new, nil
}
// Save saves the NameRecord to the database.
func (rec *NameRecord) Save() error {
if rec.Domain == "" || rec.Question == "" {
return errors.New("could not save NameRecord, missing Domain and/or Question")
}
rec.SetKey(makeNameRecordKey(rec.Domain, rec.Question))
return recordDatabase.PutNew(rec)
}

464
intel/resolve.go
View File

@@ -3,30 +3,19 @@
package intel package intel
import ( import (
"crypto/tls"
"encoding/json"
"errors"
"fmt" "fmt"
"io/ioutil"
"math/rand" "math/rand"
"net" "net"
"net/http"
"net/url"
"sort"
"strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/tevino/abool"
"github.com/Safing/safing-core/configuration" "github.com/Safing/portbase/database"
"github.com/Safing/safing-core/database" "github.com/Safing/portbase/log"
"github.com/Safing/safing-core/log" "github.com/Safing/portmaster/status"
"github.com/Safing/safing-core/network/environment"
"github.com/Safing/safing-core/network/netutils"
) )
// TODO: make resolver interface for http package // TODO: make resolver interface for http package
@@ -79,296 +68,8 @@ import (
// global -> local scopes, global // global -> local scopes, global
// special -> local scopes, local // special -> local scopes, local
type Resolver struct {
// static
Server string
ServerAddress string
IP *net.IP
Port uint16
Resolve func(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
Search *[]string
AllowedSecurityLevel int8
SkipFqdnBeforeInit string
HTTPClient *http.Client
Source string
// atomic
Initialized *abool.AtomicBool
InitLock sync.Mutex
LastFail *int64
Expires *int64
// must be locked
LockReason sync.Mutex
FailReason string
// TODO: add:
// Expiration (for server got from DHCP / ICMPv6)
// bootstrapping (first query is already sent, wait for it to either succeed or fail - think about http bootstrapping here!)
// expanded server info: type, server address, server port, options - so we do not have to parse this every time!
}
func (r *Resolver) String() string {
return r.Server
}
func (r *Resolver) Address() string {
return urlFormatAddress(r.IP, r.Port)
}
type Scope struct {
Domain string
Resolvers []*Resolver
}
var (
config = configuration.Get()
globalResolvers []*Resolver // all resolvers
localResolvers []*Resolver // all resolvers that are in site-local or link-local IP ranges
localScopes []Scope // list of scopes with a list of local resolvers that can resolve the scope
mDNSResolver *Resolver // holds a reference to the mDNS resolver
resolversLock sync.RWMutex
env = environment.NewInterface()
dupReqMap = make(map[string]*sync.Mutex)
dupReqLock sync.Mutex
)
func init() {
loadResolvers(false)
}
func indexOfResolver(server string, list []*Resolver) int {
for k, v := range list {
if v.Server == server {
return k
}
}
return -1
}
func indexOfScope(domain string, list *[]Scope) int {
for k, v := range *list {
if v.Domain == domain {
return k
}
}
return -1
}
func parseAddress(server string) (*net.IP, uint16, error) {
delimiter := strings.LastIndex(server, ":")
if delimiter < 0 {
return nil, 0, errors.New("port missing")
}
ip := net.ParseIP(strings.Trim(server[:delimiter], "[]"))
if ip == nil {
return nil, 0, errors.New("invalid IP address")
}
port, err := strconv.Atoi(server[delimiter+1:])
if err != nil || port < 1 || port > 65536 {
return nil, 0, errors.New("invalid port")
}
return &ip, uint16(port), nil
}
func urlFormatAddress(ip *net.IP, port uint16) string {
var address string
if ipv4 := ip.To4(); ipv4 != nil {
address = fmt.Sprintf("%s:%d", ipv4.String(), port)
} else {
address = fmt.Sprintf("[%s]:%d", ip.String(), port)
}
return address
}
func loadResolvers(resetResolvers bool) {
// TODO: what happens when a lot of processes want to reload at once? we do not need to run this multiple times in a short time frame.
resolversLock.Lock()
defer resolversLock.Unlock()
var newResolvers []*Resolver
configuredServersLoop:
for _, server := range config.DNSServers {
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue configuredServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
parts := strings.Split(server, "|")
if len(parts) < 2 {
log.Warningf("intel: invalid DNS server in config: %s (invalid format)", server)
continue configuredServersLoop
}
var lastFail int64
new := &Resolver{
Server: server,
ServerAddress: parts[1],
LastFail: &lastFail,
Source: "config",
Initialized: abool.NewBool(false),
}
ip, port, err := parseAddress(parts[1])
if err != nil {
new.IP = ip
new.Port = port
}
switch {
case strings.HasPrefix(server, "DNS|"):
new.Resolve = queryDNS
new.AllowedSecurityLevel = configuration.SecurityLevelFortress
case strings.HasPrefix(server, "DoH|"):
new.Resolve = queryDNSoverHTTPS
new.AllowedSecurityLevel = configuration.SecurityLevelFortress
new.SkipFqdnBeforeInit = dns.Fqdn(strings.Split(parts[1], ":")[0])
tls := &tls.Config{
// TODO: use custom random
// Rand: io.Reader,
}
tr := &http.Transport{
MaxIdleConnsPerHost: 100,
TLSClientConfig: tls,
// TODO: use custom resolver as of Go1.9
}
if len(parts) == 3 && strings.HasPrefix(parts[2], "df:") {
// activate domain fronting
tls.ServerName = parts[2][3:]
addDomainFronting(new.SkipFqdnBeforeInit, dns.Fqdn(tls.ServerName))
new.SkipFqdnBeforeInit = dns.Fqdn(tls.ServerName)
}
new.HTTPClient = &http.Client{Transport: tr}
default:
log.Warningf("intel: invalid DNS server in config: %s (not starting with a valid identifier)", server)
continue configuredServersLoop
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// add local resolvers
assignedNameservers := environment.Nameservers()
assignedServersLoop:
for _, nameserver := range assignedNameservers {
server := fmt.Sprintf("DNS|%s", urlFormatAddress(&nameserver.IP, 53))
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue assignedServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
var lastFail int64
new := &Resolver{
Server: server,
ServerAddress: urlFormatAddress(&nameserver.IP, 53),
IP: &nameserver.IP,
Port: 53,
LastFail: &lastFail,
Resolve: queryDNS,
AllowedSecurityLevel: configuration.SecurityLevelFortress,
Initialized: abool.NewBool(false),
Source: "dhcp",
}
if netutils.IPIsLocal(nameserver.IP) && len(nameserver.Search) > 0 {
// only allow searches for local resolvers
var newSearch []string
for _, value := range nameserver.Search {
newSearch = append(newSearch, fmt.Sprintf(".%s.", strings.Trim(value, ".")))
}
new.Search = &newSearch
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// save resolvers
globalResolvers = newResolvers
if len(globalResolvers) == 0 {
log.Criticalf("intel: no (valid) dns servers found in configuration and system")
}
// make list with local resolvers
localResolvers = make([]*Resolver, 0)
for _, resolver := range globalResolvers {
if resolver.IP != nil && netutils.IPIsLocal(*resolver.IP) {
localResolvers = append(localResolvers, resolver)
}
}
// add resolvers to every scope the cover
localScopes = make([]Scope, 0)
for _, resolver := range globalResolvers {
if resolver.Search != nil {
// add resolver to custom searches
for _, search := range *resolver.Search {
if search == "." {
continue
}
key := indexOfScope(search, &localScopes)
if key == -1 {
localScopes = append(localScopes, Scope{
Domain: search,
Resolvers: []*Resolver{resolver},
})
} else {
localScopes[key].Resolvers = append(localScopes[key].Resolvers, resolver)
}
}
}
}
// init mdns resolver
if mDNSResolver == nil {
cannotFail := int64(-1)
mDNSResolver = &Resolver{
Server: "mDNS",
Resolve: queryMulticastDNS,
AllowedSecurityLevel: config.DoNotUseMDNS.Level(),
Initialized: abool.NewBool(false),
Source: "static",
LastFail: &cannotFail,
}
}
// sort scopes by length
sort.Slice(localScopes,
func(i, j int) bool {
return len(localScopes[i].Domain) > len(localScopes[j].Domain)
},
)
log.Trace("intel: loaded global resolvers:")
for _, resolver := range globalResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded local resolvers:")
for _, resolver := range localResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded scopes:")
for _, scope := range localScopes {
var scopeServers []string
for _, resolver := range scope.Resolvers {
scopeServers = append(scopeServers, resolver.Server)
}
log.Tracef("intel: %s: %s", scope.Domain, strings.Join(scopeServers, ", "))
}
}
// Resolve resolves the given query for a domain and type and returns a RRCache object or nil, if the query failed. // Resolve resolves the given query for a domain and type and returns a RRCache object or nil, if the query failed.
func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { func Resolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache {
fqdn = dns.Fqdn(fqdn) fqdn = dns.Fqdn(fqdn)
// use this to time how long it takes resolve this domain // use this to time how long it takes resolve this domain
@@ -381,9 +82,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
var err error var err error
switch uint16(qtype) { switch uint16(qtype) {
case dns.TypeA: case dns.TypeA:
rr, err = dns.NewRR("localhost. 3600 IN A 127.0.0.1") rr, err = dns.NewRR("localhost. 17 IN A 127.0.0.1")
case dns.TypeAAAA: case dns.TypeAAAA:
rr, err = dns.NewRR("localhost. 3600 IN AAAA ::1") rr, err = dns.NewRR("localhost. 17 IN AAAA ::1")
default: default:
return nil return nil
} }
@@ -406,7 +107,7 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
return resolveAndCache(fqdn, qtype, securityLevel) return resolveAndCache(fqdn, qtype, securityLevel)
} }
if rrCache.Expires <= time.Now().Unix() { if rrCache.TTL <= time.Now().Unix() {
rrCache.requestingNew = true rrCache.requestingNew = true
go resolveAndCache(fqdn, qtype, securityLevel) go resolveAndCache(fqdn, qtype, securityLevel)
} }
@@ -420,17 +121,9 @@ func Resolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
return rrCache return rrCache
} }
func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { func resolveAndCache(fqdn string, qtype dns.Type, securityLevel uint8) (rrCache *RRCache) {
// log.Tracef("intel: resolving %s%s", fqdn, qtype.String()) // log.Tracef("intel: resolving %s%s", fqdn, qtype.String())
rrCache, ok := checkDomainFronting(fqdn, qtype, securityLevel)
if ok {
if rrCache == nil {
return nil
}
return rrCache
}
// dedup requests // dedup requests
dupKey := fmt.Sprintf("%s%s", fqdn, qtype.String()) dupKey := fmt.Sprintf("%s%s", fqdn, qtype.String())
dupReqLock.Lock() dupReqLock.Lock()
@@ -469,29 +162,29 @@ func resolveAndCache(fqdn string, qtype dns.Type, securityLevel int8) *RRCache {
// persist to database // persist to database
rrCache.Clean(600) rrCache.Clean(600)
rrCache.CreateWithType(fqdn, qtype) rrCache.Save()
return rrCache return rrCache
} }
func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCache { func intelligentResolve(fqdn string, qtype dns.Type, securityLevel uint8) *RRCache {
// TODO: handle being offline // TODO: handle being offline
// TODO: handle multiple network connections // TODO: handle multiple network connections
if config.Changed() { // TODO: handle these in a separate goroutine
log.Info("intel: config changed, reloading resolvers") // if config.Changed() {
loadResolvers(false) // log.Info("intel: config changed, reloading resolvers")
} else if env.NetworkChanged() { // loadResolvers(false)
log.Info("intel: network changed, reloading resolvers") // } else if env.NetworkChanged() {
loadResolvers(true) // log.Info("intel: network changed, reloading resolvers")
} // loadResolvers(true)
config.RLock() // }
defer config.RUnlock()
resolversLock.RLock() resolversLock.RLock()
defer resolversLock.RUnlock() defer resolversLock.RUnlock()
lastFailBoundary := time.Now().Unix() - config.DNSServerRetryRate lastFailBoundary := time.Now().Unix() - nameserverRetryRate()
preDottedFqdn := "." + fqdn preDottedFqdn := "." + fqdn
// resolve: // resolve:
@@ -510,11 +203,14 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
} }
} }
// check config // check config
if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) { if doNotUseMulticastDNS(securityLevel) {
return nil return nil
} }
// try mdns // try mdns
rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel) rrCache, err := queryMulticastDNS(fqdn, qtype)
if err != nil {
log.Errorf("intel: failed to query mdns: %s", err)
}
return rrCache return rrCache
} }
@@ -533,15 +229,18 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
switch { switch {
case strings.HasSuffix(preDottedFqdn, ".local."): case strings.HasSuffix(preDottedFqdn, ".local."):
// check config // check config
if config.DoNotUseMDNS.IsSetWithLevel(securityLevel) { if doNotUseMulticastDNS(securityLevel) {
return nil return nil
} }
// try mdns // try mdns
rrCache, _ := tryResolver(mDNSResolver, lastFailBoundary, fqdn, qtype, securityLevel) rrCache, err := queryMulticastDNS(fqdn, qtype)
if err != nil {
log.Errorf("intel: failed to query mdns: %s", err)
}
return rrCache return rrCache
case domainInScopes(preDottedFqdn, specialScopes): case domainInScopes(preDottedFqdn, specialScopes):
// check config // check config
if config.DoNotForwardSpecialDomains.IsSetWithLevel(securityLevel) { if doNotResolveSpecialDomains(securityLevel) {
return nil return nil
} }
// try local resolvers // try local resolvers
@@ -568,15 +267,15 @@ func intelligentResolve(fqdn string, qtype dns.Type, securityLevel int8) *RRCach
} }
func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel int8) (*RRCache, bool) { func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype dns.Type, securityLevel uint8) (*RRCache, bool) {
// skip if not allowed in current security level // skip if not allowed in current security level
if resolver.AllowedSecurityLevel < config.SecurityLevel() || resolver.AllowedSecurityLevel < securityLevel { if resolver.AllowedSecurityLevel < status.CurrentSecurityLevel() || resolver.AllowedSecurityLevel < securityLevel {
log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, config.SecurityLevel(), securityLevel) log.Tracef("intel: skipping resolver %s, because it isn't allowed to operate on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel)
return nil, false return nil, false
} }
// skip if not security level denies assigned dns servers // skip if not security level denies assigned dns servers
if config.DoNotUseAssignedDNS.IsSetWithLevel(securityLevel) && resolver.Source == "dhcp" { if doNotUseAssignedNameservers(securityLevel) && resolver.Source == "dhcp" {
log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d (%d)", resolver, config.SecurityLevel(), securityLevel, int8(config.DoNotUseAssignedDNS)) log.Tracef("intel: skipping resolver %s, because assigned nameservers are not allowed on the current security level: %d|%d", resolver, status.CurrentSecurityLevel(), securityLevel)
return nil, false return nil, false
} }
// check if failed recently // check if failed recently
@@ -606,7 +305,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype
} }
// resolve // resolve
log.Tracef("intel: trying to resolve %s%s with %s", fqdn, qtype.String(), resolver.Server) log.Tracef("intel: trying to resolve %s%s with %s", fqdn, qtype.String(), resolver.Server)
rrCache, err := resolver.Resolve(resolver, fqdn, qtype) rrCache, err := query(resolver, fqdn, qtype)
if err != nil { if err != nil {
// check if failing is disabled // check if failing is disabled
if atomic.LoadInt64(resolver.LastFail) == -1 { if atomic.LoadInt64(resolver.LastFail) == -1 {
@@ -625,7 +324,7 @@ func tryResolver(resolver *Resolver, lastFailBoundary int64, fqdn string, qtype
return rrCache, true return rrCache, true
} }
func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) { func query(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) {
q := new(dns.Msg) q := new(dns.Msg)
q.SetQuestion(fqdn, uint16(qtype)) q.SetQuestion(fqdn, uint16(qtype))
@@ -633,8 +332,7 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
var reply *dns.Msg var reply *dns.Msg
var err error var err error
for i := 0; i < 5; i++ { for i := 0; i < 5; i++ {
client := new(dns.Client) reply, _, err = resolver.clientManager.getDNSClient().Exchange(q, resolver.ServerAddress)
reply, _, err = client.Exchange(q, resolver.ServerAddress)
if err != nil { if err != nil {
// TODO: handle special cases // TODO: handle special cases
@@ -655,6 +353,8 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
} }
new := &RRCache{ new := &RRCache{
Domain: fqdn,
Question: qtype,
Answer: reply.Answer, Answer: reply.Answer,
Ns: reply.Ns, Ns: reply.Ns,
Extra: reply.Extra, Extra: reply.Extra,
@@ -663,85 +363,3 @@ func queryDNS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error)
// TODO: check if reply.Answer is valid // TODO: check if reply.Answer is valid
return new, nil return new, nil
} }
type DnsOverHttpsReply struct {
Status uint32
Truncated bool `json:"TC"`
Answer []DohRR
Additional []DohRR
}
type DohRR struct {
Name string `json:"name"`
Qtype uint16 `json:"type"`
TTL uint32 `json:"TTL"`
Data string `json:"data"`
}
func queryDNSoverHTTPS(resolver *Resolver, fqdn string, qtype dns.Type) (*RRCache, error) {
// API documentation: https://developers.google.com/speed/public-dns/docs/dns-over-https
payload := url.Values{}
payload.Add("name", fqdn)
payload.Add("type", strconv.Itoa(int(qtype)))
payload.Add("edns_client_subnet", "0.0.0.0/0")
// TODO: add random - only use upper- and lower-case letters, digits, hyphen, period, underscore and tilde
// payload.Add("random_padding", "")
resp, err := resolver.HTTPClient.Get(fmt.Sprintf("https://%s/resolve?%s", resolver.ServerAddress, payload.Encode()))
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: http error: %s", fqdn, qtype.String(), err)
// TODO: handle special cases
// 1. connect: network is unreachable
// intel: resolver DoH|dns.google.com:443|df:www.google.com failed (resolving discovery-v4-4.syncthing.net.A failed: http error: Get https://dns.google.com:443/resolve?edns_client_subnet=0.0.0.0%2F0&name=discovery-v4-4.syncthing.net.&type=1: dial tcp [2a00:1450:4001:819::2004]:443: connect: network is unreachable), moving to next
// 2. timeout
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("resolving %s%s failed: request was unsuccessful, got code %d", fqdn, qtype.String(), resp.StatusCode)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: error reading response body: %s", fqdn, qtype.String(), err)
}
var reply DnsOverHttpsReply
err = json.Unmarshal(body, &reply)
if err != nil {
return nil, fmt.Errorf("resolving %s%s failed: error parsing response body: %s", fqdn, qtype.String(), err)
}
if reply.Status != 0 {
// this happens if there is a server error (e.g. DNSSEC fail), ignore for now
// TODO: do something more intelligent
}
new := new(RRCache)
// TODO: handle TXT records
for _, entry := range reply.Answer {
rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data))
if err != nil {
log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)
continue
}
new.Answer = append(new.Answer, rr)
}
for _, entry := range reply.Additional {
rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data))
if err != nil {
log.Warningf("intel: resolving %s%s failed: failed to parse record to DNS: %s %d IN %s %s", fqdn, qtype.String(), entry.Name, entry.TTL, dns.Type(entry.Qtype).String(), entry.Data)
continue
}
new.Extra = append(new.Extra, rr)
}
return new, nil
}
// TODO: implement T-DNS: DNS over TCP/TLS
// server list: https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers

292
intel/resolver.go Normal file
View File

@@ -0,0 +1,292 @@
package intel
import (
"errors"
"fmt"
"net"
"sort"
"strconv"
"strings"
"sync"
"github.com/miekg/dns"
"github.com/tevino/abool"
"github.com/Safing/portbase/log"
"github.com/Safing/portmaster/network/environment"
"github.com/Safing/portmaster/network/netutils"
"github.com/Safing/portmaster/status"
)
// Resolver holds information about an active resolver.
type Resolver struct {
// static
Server string
ServerType string
ServerAddress string
ServerIP net.IP
ServerPort uint16
VerifyDomain string
Source string
clientManager *clientManager
Search *[]string
AllowedSecurityLevel uint8
SkipFqdnBeforeInit string
// atomic
Initialized *abool.AtomicBool
InitLock sync.Mutex
LastFail *int64
Expires *int64
// must be locked
LockReason sync.Mutex
FailReason string
// TODO: add:
// Expiration (for server got from DHCP / ICMPv6)
// bootstrapping (first query is already sent, wait for it to either succeed or fail - think about http bootstrapping here!)
// expanded server info: type, server address, server port, options - so we do not have to parse this every time!
}
func (r *Resolver) String() string {
return r.Server
}
// Scope defines a domain scope and which resolvers can resolve it.
type Scope struct {
Domain string
Resolvers []*Resolver
}
var (
globalResolvers []*Resolver // all resolvers
localResolvers []*Resolver // all resolvers that are in site-local or link-local IP ranges
localScopes []*Scope // list of scopes with a list of local resolvers that can resolve the scope
resolversLock sync.RWMutex
env = environment.NewInterface()
dupReqMap = make(map[string]*sync.Mutex)
dupReqLock sync.Mutex
)
func indexOfResolver(server string, list []*Resolver) int {
for k, v := range list {
if v.Server == server {
return k
}
}
return -1
}
func indexOfScope(domain string, list []*Scope) int {
for k, v := range list {
if v.Domain == domain {
return k
}
}
return -1
}
func parseAddress(server string) (net.IP, uint16, error) {
delimiter := strings.LastIndex(server, ":")
if delimiter < 0 {
return nil, 0, errors.New("port missing")
}
ip := net.ParseIP(strings.Trim(server[:delimiter], "[]"))
if ip == nil {
return nil, 0, errors.New("invalid IP address")
}
port, err := strconv.Atoi(server[delimiter+1:])
if err != nil || port < 1 || port > 65536 {
return nil, 0, errors.New("invalid port")
}
return ip, uint16(port), nil
}
func urlFormatAddress(ip net.IP, port uint16) string {
var address string
if ipv4 := ip.To4(); ipv4 != nil {
address = fmt.Sprintf("%s:%d", ipv4.String(), port)
} else {
address = fmt.Sprintf("[%s]:%d", ip.String(), port)
}
return address
}
func loadResolvers(resetResolvers bool) {
// TODO: what happens when a lot of processes want to reload at once? we do not need to run this multiple times in a short time frame.
resolversLock.Lock()
defer resolversLock.Unlock()
var newResolvers []*Resolver
configuredServersLoop:
for _, server := range configuredNameServers() {
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue configuredServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
parts := strings.Split(server, "|")
if len(parts) < 2 {
log.Warningf("intel: nameserver format invalid: %s", server)
continue configuredServersLoop
}
ip, port, err := parseAddress(parts[1])
if err != nil && strings.ToLower(parts[0]) != "https" {
log.Warningf("intel: nameserver (%s) address invalid: %s", server, err)
continue configuredServersLoop
}
var lastFail int64
new := &Resolver{
Server: server,
ServerType: parts[0],
ServerAddress: parts[1],
ServerIP: ip,
ServerPort: port,
LastFail: &lastFail,
Source: "config",
Initialized: abool.NewBool(false),
}
switch strings.ToLower(parts[0]) {
case "dns":
new.clientManager = newDNSClientManager(new)
case "tcp":
new.clientManager = newTCPClientManager(new)
case "tls":
new.AllowedSecurityLevel = status.SecurityLevelFortress
if len(parts) < 3 {
log.Warningf("intel: nameserver missing verification domain as third parameter: %s", server)
continue configuredServersLoop
}
new.VerifyDomain = parts[2]
new.clientManager = newTLSClientManager(new)
case "https":
new.AllowedSecurityLevel = status.SecurityLevelFortress
new.SkipFqdnBeforeInit = dns.Fqdn(strings.Split(parts[1], ":")[0])
if len(parts) > 2 {
new.VerifyDomain = parts[2]
}
new.clientManager = newHTTPSClientManager(new)
default:
log.Warningf("intel: nameserver (%s) type invalid: %s", server, parts[0])
continue configuredServersLoop
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// add local resolvers
assignedNameservers := environment.Nameservers()
assignedServersLoop:
for _, nameserver := range assignedNameservers {
server := fmt.Sprintf("dns|%s", urlFormatAddress(nameserver.IP, 53))
key := indexOfResolver(server, newResolvers)
if key >= 0 {
continue assignedServersLoop
}
key = indexOfResolver(server, globalResolvers)
if resetResolvers || key == -1 {
var lastFail int64
new := &Resolver{
Server: server,
ServerType: "dns",
ServerAddress: urlFormatAddress(nameserver.IP, 53),
ServerIP: nameserver.IP,
ServerPort: 53,
LastFail: &lastFail,
Source: "dhcp",
Initialized: abool.NewBool(false),
AllowedSecurityLevel: status.SecurityLevelSecure,
}
new.clientManager = newDNSClientManager(new)
if netutils.IPIsLocal(nameserver.IP) && len(nameserver.Search) > 0 {
// only allow searches for local resolvers
var newSearch []string
for _, value := range nameserver.Search {
newSearch = append(newSearch, fmt.Sprintf(".%s.", strings.Trim(value, ".")))
}
new.Search = &newSearch
}
newResolvers = append(newResolvers, new)
} else {
newResolvers = append(newResolvers, globalResolvers[key])
}
}
// save resolvers
globalResolvers = newResolvers
if len(globalResolvers) == 0 {
log.Criticalf("intel: no (valid) dns servers found in configuration and system")
}
// make list with local resolvers
localResolvers = make([]*Resolver, 0)
for _, resolver := range globalResolvers {
if resolver.ServerIP != nil && netutils.IPIsLocal(resolver.ServerIP) {
localResolvers = append(localResolvers, resolver)
}
}
// add resolvers to every scope the cover
localScopes = make([]*Scope, 0)
for _, resolver := range globalResolvers {
if resolver.Search != nil {
// add resolver to custom searches
for _, search := range *resolver.Search {
if search == "." {
continue
}
key := indexOfScope(search, localScopes)
if key == -1 {
localScopes = append(localScopes, &Scope{
Domain: search,
Resolvers: []*Resolver{resolver},
})
} else {
localScopes[key].Resolvers = append(localScopes[key].Resolvers, resolver)
}
}
}
}
// sort scopes by length
sort.Slice(localScopes,
func(i, j int) bool {
return len(localScopes[i].Domain) > len(localScopes[j].Domain)
},
)
log.Trace("intel: loaded global resolvers:")
for _, resolver := range globalResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded local resolvers:")
for _, resolver := range localResolvers {
log.Tracef("intel: %s", resolver.Server)
}
log.Trace("intel: loaded scopes:")
for _, scope := range localScopes {
var scopeServers []string
for _, resolver := range scope.Resolvers {
scopeServers = append(scopeServers, resolver.Server)
}
log.Tracef("intel: %s: %s", scope.Domain, strings.Join(scopeServers, ", "))
}
}

164
intel/rrcache.go Normal file
View File

@@ -0,0 +1,164 @@
// Copyright Safing ICS Technologies GmbH. Use of this source code is governed by the AGPL license that can be found in the LICENSE file.
package intel
import (
"net"
"time"
"github.com/miekg/dns"
)
// RRCache is used to cache DNS data
type RRCache struct {
Domain string
Question dns.Type
Answer []dns.RR
Ns []dns.RR
Extra []dns.RR
TTL int64
updated int64
servedFromCache bool
requestingNew bool
}
// Clean sets all TTLs to 17 and sets cache expiry with specified minimum.
func (m *RRCache) Clean(minExpires uint32) {
var lowestTTL uint32 = 0xFFFFFFFF
var header *dns.RR_Header
// set TTLs to 17
// TODO: double append? is there something more elegant?
for _, rr := range append(m.Answer, append(m.Ns, m.Extra...)...) {
header = rr.Header()
if lowestTTL > header.Ttl {
lowestTTL = header.Ttl
}
header.Ttl = 17
}
// TTL must be at least minExpires
if lowestTTL < minExpires {
lowestTTL = minExpires
}
m.TTL = time.Now().Unix() + int64(lowestTTL)
}
// ExportAllARecords return of a list of all A and AAAA IP addresses.
func (m *RRCache) ExportAllARecords() (ips []net.IP) {
for _, rr := range m.Answer {
if rr.Header().Class != dns.ClassINET {
continue
}
switch rr.Header().Rrtype {
case dns.TypeA:
aRecord, ok := rr.(*dns.A)
if ok {
ips = append(ips, aRecord.A)
}
case dns.TypeAAAA:
aaaaRecord, ok := rr.(*dns.AAAA)
if ok {
ips = append(ips, aaaaRecord.AAAA)
}
}
}
return
}
// ToNameRecord converts the RRCache to a NameRecord for cleaner persistence.
func (m *RRCache) ToNameRecord() *NameRecord {
new := &NameRecord{
Domain: m.Domain,
Question: m.Question.String(),
TTL: m.TTL,
}
// stringify RR entries
for _, entry := range m.Answer {
new.Answer = append(new.Answer, entry.String())
}
for _, entry := range m.Ns {
new.Ns = append(new.Ns, entry.String())
}
for _, entry := range m.Extra {
new.Extra = append(new.Extra, entry.String())
}
return new
}
// Save saves the RRCache to the database as a NameRecord.
func (m *RRCache) Save() error {
return m.ToNameRecord().Save()
}
// GetRRCache tries to load the corresponding NameRecord from the database and convert it.
func GetRRCache(domain string, question dns.Type) (*RRCache, error) {
var m RRCache
rr := &RRCache{
Domain: domain,
Question: question,
}
nameRecord, err := GetNameRecord(domain, question.String())
if err != nil {
return nil, err
}
rr.TTL = nameRecord.TTL
for _, entry := range nameRecord.Answer {
rr, err := dns.NewRR(entry)
if err == nil {
m.Answer = append(m.Answer, rr)
}
}
for _, entry := range nameRecord.Ns {
rr, err := dns.NewRR(entry)
if err == nil {
m.Ns = append(m.Ns, rr)
}
}
for _, entry := range nameRecord.Extra {
rr, err := dns.NewRR(entry)
if err == nil {
m.Extra = append(m.Extra, rr)
}
}
m.servedFromCache = true
return &m, nil
}
// ServedFromCache marks the RRCache as served from cache.
func (m *RRCache) ServedFromCache() bool {
return m.servedFromCache
}
// RequestingNew informs that it has expired and new RRs are being fetched.
func (m *RRCache) RequestingNew() bool {
return m.requestingNew
}
// Flags formats ServedFromCache and RequestingNew to a condensed, flag-like format.
func (m *RRCache) Flags() string {
switch {
case m.servedFromCache && m.requestingNew:
return " [CR]"
case m.servedFromCache:
return " [C]"
case m.requestingNew:
return " [R]" // should never enter this state, but let's leave it here, just in case
default:
return ""
}
}
// IsNXDomain returnes whether the result is nxdomain.
func (m *RRCache) IsNXDomain() bool {
return len(m.Answer) == 0
}

9
intel/special.go
View File

@@ -5,7 +5,7 @@ package intel
import "strings" import "strings"
var ( var (
localReverseScopes = &[]string{ localReverseScopes = []string{
".10.in-addr.arpa.", ".10.in-addr.arpa.",
".16.172.in-addr.arpa.", ".16.172.in-addr.arpa.",
".17.172.in-addr.arpa.", ".17.172.in-addr.arpa.",
@@ -31,7 +31,8 @@ var (
".b.e.f.ip6.arpa.", ".b.e.f.ip6.arpa.",
} }
specialScopes = &[]string{ // RFC6761, RFC7686
specialScopes = []string{
".example.", ".example.",
".example.com.", ".example.com.",
".example.net.", ".example.net.",
@@ -42,8 +43,8 @@ var (
} }
) )
func domainInScopes(fqdn string, list *[]string) bool { func domainInScopes(fqdn string, list []string) bool {
for _, scope := range *list { for _, scope := range list {
if strings.HasSuffix(fqdn, scope) { if strings.HasSuffix(fqdn, scope) {
return true return true
} }