Merge branch 'develop' into develop

resolver/config.go

@@ -112,43 +112,42 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 		ExpertiseLevel:  config.ExpertiseLevelUser,
 		ReleaseLevel:    config.ReleaseLevelStable,
 		DefaultValue:    defaultNameServers,
-		ValidationRegex: fmt.Sprintf("^(%s|%s|%s)://.*", ServerTypeDoT, ServerTypeDNS, ServerTypeTCP),
+		ValidationRegex: fmt.Sprintf("^(%s|%s|%s|%s|%s|%s)://.*", ServerTypeDoT, ServerTypeDoH, ServerTypeDNS, ServerTypeTCP, HTTPSProtocol, TLSProtocol),
 		ValidationFunc:  validateNameservers,
 		Annotations: config.Annotations{
 			config.DisplayHintAnnotation:  config.DisplayHintOrdered,
 			config.DisplayOrderAnnotation: cfgOptionNameServersOrder,
 			config.CategoryAnnotation:     "Servers",
 			config.QuickSettingsAnnotation: []config.QuickSetting{
-				{
+			{
 					Name:   "Cloudflare (with Malware Filter)",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://1.1.1.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"dot://cloudflare-dns.com?ip=1.1.1.2&name=Cloudflare&blockedif=zeroip",
-						"dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"dot://cloudflare-dns.com?ip=1.0.0.2&name=Cloudflare&blockedif=zeroip",
 					},
 				},
 				{
 					Name:   "Quad9",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://9.9.9.9:853?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"dot://dns.quad9.net?ip=9.9.9.9&name=Quad9&blockedif=empty",
-						"dot://149.112.112.112:853?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"dot://dns.quad9.net?ip=149.112.112.112&name=Quad9&blockedif=empty",
 					},
 				},
 				{
 					Name:   "AdGuard",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://94.140.14.14:853?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"dot://dns.adguard.com?ip=94.140.14.14&name=AdGuard&blockedif=zeroip",
-						"dot://94.140.15.15:853?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"dot://dns.adguard.com?ip=94.140.15.15&name=AdGuard&blockedif=zeroip",
 					},
 				},
 				{
 					Name:   "Foundation for Applied Privacy",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://94.130.106.88:853?verify=dot1.applied-privacy.net&name=AppliedPrivacy",
+						"dot://dot1.applied-privacy.net?ip=94.130.106.88&name=AppliedPrivacy",
-						"dot://94.130.106.88:443?verify=dot1.applied-privacy.net&name=AppliedPrivacy",
 					},
 				},
 			},

resolver/resolver-https.go

@@ -0,0 +1,120 @@
+package resolver
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+// HTTPSResolver is a resolver using just a single tcp connection with pipelining.
+type HTTPSResolver struct {
+	BasicResolverConn
+	Client *http.Client
+}
+
+// HTTPSQuery holds the query information for a hTTPSResolverConn.
+type HTTPSQuery struct {
+	Query    *Query
+	Response chan *dns.Msg
+}
+
+// MakeCacheRecord creates an RRCache record from a reply.
+func (tq *HTTPSQuery) MakeCacheRecord(reply *dns.Msg, resolverInfo *ResolverInfo) *RRCache {
+	return &RRCache{
+		Domain:   tq.Query.FQDN,
+		Question: tq.Query.QType,
+		RCode:    reply.Rcode,
+		Answer:   reply.Answer,
+		Ns:       reply.Ns,
+		Extra:    reply.Extra,
+		Resolver: resolverInfo.Copy(),
+	}
+}
+
+// NewHTTPSResolver returns a new HTTPSResolver.
+func NewHTTPSResolver(resolver *Resolver) *HTTPSResolver {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+			ServerName: resolver.Info.Domain,
+			// TODO: use portbase rng
+		},
+		IdleConnTimeout: 3 * time.Minute,
+	}
+
+	client := &http.Client{Transport: tr}
+	newResolver := &HTTPSResolver{
+		BasicResolverConn: BasicResolverConn{
+			resolver: resolver,
+		},
+		Client: client,
+	}
+	newResolver.BasicResolverConn.init()
+	return newResolver
+}
+
+// Query executes the given query against the resolver.
+func (hr *HTTPSResolver) Query(ctx context.Context, q *Query) (*RRCache, error) {
+	dnsQuery := new(dns.Msg)
+	dnsQuery.SetQuestion(q.FQDN, uint16(q.QType))
+
+	// Pack query and convert to base64 string
+	buf, err := dnsQuery.Pack()
+	if err != nil {
+		return nil, err
+	}
+	b64dns := base64.RawStdEncoding.EncodeToString(buf)
+
+	// Build and execute http reuqest
+	url := &url.URL{
+		Scheme:     "https",
+		Host:       hr.resolver.ServerAddress,
+		Path:       hr.resolver.Path,
+		ForceQuery: true,
+		RawQuery:   fmt.Sprintf("dns=%s", b64dns),
+	}
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := hr.Client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Try to read the result
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	reply := new(dns.Msg)
+
+	err = reply.Unpack(body)
+	if err != nil {
+		return nil, err
+	}
+
+	newRecord := &RRCache{
+		Domain:   q.FQDN,
+		Question: q.QType,
+		RCode:    reply.Rcode,
+		Answer:   reply.Answer,
+		Ns:       reply.Ns,
+		Extra:    reply.Extra,
+		Resolver: hr.resolver.Info.Copy(),
+	}
+
+	// TODO: check if reply.Answer is valid
+	return newRecord, nil
+}

resolver/resolver-tcp.go

@@ -99,7 +99,7 @@ func (tr *TCPResolver) UseTLS() *TCPResolver {
 	tr.dnsClient.Net = "tcp-tls"
 	tr.dnsClient.TLSConfig = &tls.Config{
 		MinVersion: tls.VersionTLS12,
-		ServerName: tr.resolver.VerifyDomain,
+		ServerName: tr.resolver.Info.Domain,
 		// TODO: use portbase rng
 	}
 	return tr

resolver/resolver.go

@@ -30,6 +30,12 @@ const (
 	ServerSourceEnv             = "env"
 )
 
+// DNS Resolver alias
+const (
+	HTTPSProtocol = "https"
+	TLSProtocol   = "tls"
+)
+
 // FailThreshold is amount of errors a resolvers must experience in order to be regarded as failed.
 var FailThreshold = 20
 
@@ -61,9 +67,9 @@ type Resolver struct {
 	UpstreamBlockDetection string
 
 	// Special Options
-	VerifyDomain string
+	Search     []string
-	Search       []string
+	SearchOnly bool
-	SearchOnly   bool
+	Path       string
 
 	// logic interface
 	Conn ResolverConn `json:"-"`
@@ -87,6 +93,9 @@ type ResolverInfo struct { //nolint:golint,maligned // TODO
 	// IP is the IP address of the resolver
 	IP net.IP
 
+	// Domain of the dns server if it has one
+	Domain string
+
 	// IPScope is the network scope of the IP address.
 	IPScope netutils.IPScope
 
@@ -107,6 +116,20 @@ func (info *ResolverInfo) ID() string {
 			info.id = ServerTypeMDNS
 		case ServerTypeEnv:
 			info.id = ServerTypeEnv
+		case ServerTypeDoH:
+			info.id = fmt.Sprintf(
+				"https://%s:%d#%s",
+				info.Domain,
+				info.Port,
+				info.Source,
+			)
+		case ServerTypeDoT:
+			info.id = fmt.Sprintf(
+				"dot://%s:%d#%s",
+				info.Domain,
+				info.Port,
+				info.Source,
+			)
 		default:
 			info.id = fmt.Sprintf(
 				"%s://%s:%d#%s",
@@ -135,6 +158,12 @@ func (info *ResolverInfo) DescriptiveName() string {
 			info.Name,
 			info.ID(),
 		)
+	case info.Domain != "":
+		return fmt.Sprintf(
+			"%s (%s)",
+			info.Domain,
+			info.ID(),
+		)
 	default:
 		return fmt.Sprintf(
 			"%s (%s)",
@@ -155,6 +184,7 @@ func (info *ResolverInfo) Copy() *ResolverInfo {
 		Type:    info.Type,
 		Source:  info.Source,
 		IP:      info.IP,
+		Domain:  info.Domain,
 		IPScope: info.IPScope,
 		Port:    info.Port,
 		id:      info.id,

resolver/resolvers.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/net/publicsuffix"
 
+	"github.com/miekg/dns"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/utils"
 	"github.com/safing/portmaster/netenv"
@@ -29,9 +30,11 @@ type Scope struct {
 const (
 	parameterName       = "name"
 	parameterVerify     = "verify"
+	parameterIP         = "ip"
 	parameterBlockedIf  = "blockedif"
 	parameterSearch     = "search"
 	parameterSearchOnly = "search-only"
+	parameterPath       = "path"
 )
 
 var (
@@ -40,7 +43,8 @@ var (
 	systemResolvers       []*Resolver          // all resolvers that were assigned by the system
 	localScopes           []*Scope             // list of scopes with a list of local resolvers that can resolve the scope
 	activeResolvers       map[string]*Resolver // lookup map of all resolvers
-	currentResolverConfig []string             // current active resolver config, to detect changes
+  resolverInitDomains   map[string]struct{}  // a set with all domains of the dns resolvers
+
 	resolversLock         sync.RWMutex
 )
 
@@ -80,6 +84,8 @@ func resolverConnFactory(resolver *Resolver) ResolverConn {
 		return NewTCPResolver(resolver)
 	case ServerTypeDoT:
 		return NewTCPResolver(resolver).UseTLS()
+	case ServerTypeDoH:
+		return NewHTTPSResolver(resolver)
 	case ServerTypeDNS:
 		return NewPlainResolver(resolver)
 	default:
@@ -93,93 +99,64 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 		return nil, false, err
 	}
 
+	if resolverInitDomains == nil {
+		resolverInitDomains = make(map[string]struct{})
+	}
+
 	switch u.Scheme {
-	case ServerTypeDNS, ServerTypeDoT, ServerTypeTCP:
+	case ServerTypeDNS, ServerTypeDoT, ServerTypeDoH, ServerTypeTCP:
+	case HTTPSProtocol:
+		u.Scheme = ServerTypeDoH
+	case TLSProtocol:
+		u.Scheme = ServerTypeDoT
 	default:
 		return nil, false, fmt.Errorf("DNS resolver scheme %q invalid", u.Scheme)
 	}
 
-	ip := net.ParseIP(u.Hostname())
-	if ip == nil {
-		return nil, false, fmt.Errorf("resolver IP %q invalid", u.Hostname())
-	}
-
-	// Add default port for scheme if it is missing.
-	var port uint16
-	hostPort := u.Port()
-	switch {
-	case hostPort != "":
-		parsedPort, err := strconv.ParseUint(hostPort, 10, 16)
-		if err != nil {
-			return nil, false, fmt.Errorf("resolver port %q invalid", u.Port())
-		}
-		port = uint16(parsedPort)
-	case u.Scheme == ServerTypeDNS, u.Scheme == ServerTypeTCP:
-		port = 53
-	case u.Scheme == ServerTypeDoH:
-		port = 443
-	case u.Scheme == ServerTypeDoT:
-		port = 853
-	default:
-		return nil, false, fmt.Errorf("missing port in %q", u.Host)
-	}
-
-	scope := netutils.GetIPScope(ip)
-	// Skip localhost resolvers from the OS, but not if configured.
-	if scope.IsLocalhost() && source == ServerSourceOperatingSystem {
-		return nil, true, nil // skip
-	}
-
-	// Get parameters and check if keys exist.
 	query := u.Query()
-	for key := range query {
-		switch key {
-		case parameterName,
-			parameterVerify,
-			parameterBlockedIf,
-			parameterSearch,
-			parameterSearchOnly:
-			// Known key, continue.
-		default:
-			// Unknown key, abort.
-			return nil, false, fmt.Errorf(`unknown parameter "%s"`, key)
-		}
-	}
 
-	// Check domain verification config.
+	// Create Resolver object
-	verifyDomain := query.Get(parameterVerify)
-	if verifyDomain != "" && u.Scheme != ServerTypeDoT {
-		return nil, false, fmt.Errorf("domain verification only supported in DOT")
-	}
-	if verifyDomain == "" && u.Scheme == ServerTypeDoT {
-		return nil, false, fmt.Errorf("DOT must have a verify query parameter set")
-	}
-
-	// Check block detection type.
-	blockType := query.Get(parameterBlockedIf)
-	if blockType == "" {
-		blockType = BlockDetectionZeroIP
-	}
-	switch blockType {
-	case BlockDetectionDisabled, BlockDetectionEmptyAnswer, BlockDetectionRefused, BlockDetectionZeroIP:
-	default:
-		return nil, false, fmt.Errorf("invalid value for upstream block detection (blockedif=)")
-	}
-
-	// Build resolver.
 	newResolver := &Resolver{
 		ConfigURL: resolverURL,
 		Info: &ResolverInfo{
 			Name:    query.Get(parameterName),
 			Type:    u.Scheme,
 			Source:  source,
-			IP:      ip,
+			IP:      nil,
-			IPScope: scope,
+			Domain:  "",
-			Port:    port,
+			IPScope: netutils.Global,
+			Port:    0,
 		},
-		ServerAddress:          net.JoinHostPort(ip.String(), strconv.Itoa(int(port))),
+		ServerAddress:          "",
-		VerifyDomain:           verifyDomain,
+		Path:                   u.Path, // Used for DoH
-		UpstreamBlockDetection: blockType,
+		UpstreamBlockDetection: "",
+	}
+
+	// Get parameters and check if keys exist.
+	err = checkAndSetResolverParamters(u, newResolver)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// Check block detection type.
+	newResolver.UpstreamBlockDetection = query.Get(parameterBlockedIf)
+	if newResolver.UpstreamBlockDetection == "" {
+		newResolver.UpstreamBlockDetection = BlockDetectionZeroIP
+	}
+
+	switch newResolver.UpstreamBlockDetection {
+	case BlockDetectionDisabled, BlockDetectionEmptyAnswer, BlockDetectionRefused, BlockDetectionZeroIP:
+	default:
+		return nil, false, fmt.Errorf("invalid value for upstream block detection (blockedif=)")
+	}
+
+	// Get ip scope if we have ip
+	if newResolver.Info.IP != nil {
+		newResolver.Info.IPScope = netutils.GetIPScope(newResolver.Info.IP)
+		// Skip localhost resolvers from the OS, but not if configured.
+		if newResolver.Info.IPScope.IsLocalhost() && source == ServerSourceOperatingSystem {
+			return nil, true, nil // skip
+		}
 	}
 
 	// Parse search domains.
@@ -206,6 +183,108 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 	return newResolver, false, nil
 }
 
+func checkAndSetResolverParamters(u *url.URL, resolver *Resolver) error {
+	// Check if we are using domain name and if it's in a valid scheme
+	ip := net.ParseIP(u.Hostname())
+	hostnameIsDomaion := (ip == nil)
+	if ip == nil && u.Scheme != ServerTypeDoH && u.Scheme != ServerTypeDoT {
+		return fmt.Errorf("resolver IP %q is invalid", u.Hostname())
+	}
+
+	// Add default port for scheme if it is missing.
+	port, err := parsePortFromURL(u)
+	if err != nil {
+		return err
+	}
+	resolver.Info.Port = port
+	resolver.Info.IP = ip
+
+	query := u.Query()
+
+	for key := range query {
+		switch key {
+		case parameterName,
+			parameterVerify,
+			parameterIP,
+			parameterBlockedIf,
+			parameterSearch,
+			parameterSearchOnly,
+			parameterPath:
+			// Known key, continue.
+		default:
+			// Unknown key, abort.
+			return fmt.Errorf(`unknown parameter "%q"`, key)
+		}
+	}
+
+	resolver.Info.Domain = query.Get(parameterVerify)
+	paramterServerIP := query.Get(parameterIP)
+
+	if u.Scheme == ServerTypeDoT || u.Scheme == ServerTypeDoH {
+		// Check if IP and Domain are set correctly
+		switch {
+		case hostnameIsDomaion && resolver.Info.Domain != "":
+			return fmt.Errorf("cannot set the domain name via both the hostname in the URL and the verify parameter")
+		case !hostnameIsDomaion && resolver.Info.Domain == "":
+			return fmt.Errorf("verify parameter must be set when using ip as domain")
+		case !hostnameIsDomaion && paramterServerIP != "":
+			return fmt.Errorf("cannot set the IP address via both the hostname in the URL and the ip parameter")
+		}
+
+		// Parse and set IP and Domain to the resolver
+		switch {
+		case hostnameIsDomaion && paramterServerIP != "": // domain and ip as parameter
+			resolver.Info.IP = net.ParseIP(paramterServerIP)
+			resolver.ServerAddress = net.JoinHostPort(paramterServerIP, strconv.Itoa(int(resolver.Info.Port)))
+			resolver.Info.Domain = u.Hostname()
+		case !hostnameIsDomaion && resolver.Info.Domain != "": // ip and domain as parameter
+			resolver.ServerAddress = net.JoinHostPort(ip.String(), strconv.Itoa(int(resolver.Info.Port)))
+		case hostnameIsDomaion && resolver.Info.Domain == "" && paramterServerIP == "": // only domain
+			resolver.Info.Domain = u.Hostname()
+			resolver.ServerAddress = net.JoinHostPort(resolver.Info.Domain, strconv.Itoa(int(port)))
+		}
+
+		if ip == nil {
+			resolverInitDomains[dns.Fqdn(resolver.Info.Domain)] = struct{}{}
+		}
+
+	} else {
+		if resolver.Info.Domain != "" {
+			return fmt.Errorf("domain verification is only supported by DoT and DoH servers")
+		}
+		resolver.ServerAddress = net.JoinHostPort(ip.String(), strconv.Itoa(int(resolver.Info.Port)))
+	}
+
+	return nil
+}
+
+func parsePortFromURL(url *url.URL) (uint16, error) {
+	var port uint16
+	hostPort := url.Port()
+	if hostPort != "" {
+		// There is a port in the url
+		parsedPort, err := strconv.ParseUint(hostPort, 10, 16)
+		if err != nil {
+			return 0, fmt.Errorf("invalid port %q", url.Port())
+		}
+		port = uint16(parsedPort)
+	} else {
+		// set the default port for the protocol
+		switch {
+		case url.Scheme == ServerTypeDNS, url.Scheme == ServerTypeTCP:
+			port = 53
+		case url.Scheme == ServerTypeDoH:
+			port = 443
+		case url.Scheme == ServerTypeDoT:
+			port = 853
+		default:
+			return 0, fmt.Errorf("cannot determine port for %q", url.Scheme)
+		}
+	}
+
+	return port, nil
+}
+
 func configureSearchDomains(resolver *Resolver, searches []string, hardfail bool) error {
 	resolver.Search = make([]string, 0, len(searches))
 

resolver/scopes.go

@@ -220,6 +220,13 @@ addNextResolver:
 			}
 		}
 
+		// the domains from the configured resolvers should not be resolved with the same resolvers
+		if resolver.Info.Source == ServerSourceConfigured && resolver.Info.IP == nil {
+			if _, ok := resolverInitDomains[q.FQDN]; ok {
+				continue addNextResolver
+			}
+		}
+
 		// add compliant and unique resolvers to selected resolvers
 		selected = append(selected, resolver)
 	}