DoH domain resolving and parsing improvments

resolver/config.go

@@ -111,7 +111,7 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 		ExpertiseLevel:  config.ExpertiseLevelExpert,
 		ReleaseLevel:    config.ReleaseLevelStable,
 		DefaultValue:    defaultNameServers,
-		ValidationRegex: fmt.Sprintf("^(%s|%s|%s|%s)://.*", ServerTypeDoT, ServerTypeDoH, ServerTypeDNS, ServerTypeTCP),
+		ValidationRegex: fmt.Sprintf("^(%s|%s|%s|%s|%s)://.*", ServerTypeDoT, ServerTypeDoH, ServerTypeDNS, ServerTypeTCP, HttpsProtocol),
 		ValidationFunc:  validateNameservers,
 		Annotations: config.Annotations{
 			config.DisplayHintAnnotation:  config.DisplayHintOrdered,
@@ -122,56 +122,62 @@ The format is: "protocol://ip:port?parameter=value&parameter=value"
 					Name:   "Quad9",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://9.9.9.9:853?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"dot://dns.quad9.net?ip=9.9.9.9&verify=&name=Quad9&blockedif=empty",
-						"dot://149.112.112.112:853?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"dot://dns.quad9.net?ip=149.112.112.112&verify=&name=Quad9&blockedif=empty",
 					},
 				},
 				{
 					Name:   "Quad9 DoH",
 					Action: config.QuickReplace,
 					Value: []string{
-						"doh://149.112.112.112:443?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"https://dns.quad9.net/dns-query?ip=9.9.9.9&name=Quad9",
-						"doh://9.9.9.9:443?verify=dns.quad9.net&name=Quad9&blockedif=empty",
+						"https://dns.quad9.net/dns-query?ip=149.112.112.112&name=Quad9",
 					},
 				},
 				{
 					Name:   "AdGuard",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://94.140.14.14:853?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"dot://dns.adguard.com?ip=94.140.14.14&name=AdGuard&blockedif=zeroip",
-						"dot://94.140.15.15:853?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"dot://dns.adguard.com?ip=94.140.15.15&name=AdGuard&blockedif=zeroip",
 					},
 				},
 				{
 					Name:   "AdGuard DoH",
 					Action: config.QuickReplace,
 					Value: []string{
-						"doh://94.140.14.14:443?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"https://dns.adguard.com/dns-query?ip=94.140.14.14&name=AdGuard",
-						"doh://94.140.15.15:443?verify=dns.adguard.com&name=AdGuard&blockedif=zeroip",
+						"https://dns.adguard.com/dns-query?ip=94.140.15.15&name=AdGuard",
 					},
 				},
 				{
 					Name:   "Foundation for Applied Privacy",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://94.130.106.88:853?verify=dot1.applied-privacy.net&name=AppliedPrivacy",
+						"dot://dot1.applied-privacy.net?ip=94.130.106.88&name=AppliedPrivacy",
-						"dot://94.130.106.88:443?verify=dot1.applied-privacy.net&name=AppliedPrivacy",
+					},
+				},
+				{
+					Name:   "Foundation for Applied Privacy DoH",
+					Action: config.QuickReplace,
+					Value: []string{
+						"https://dot1.applied-privacy.net/query?ip=94.130.106.88&name=AppliedPrivacy",
 					},
 				},
 				{
 					Name:   "Cloudflare (with Malware Filter)",
 					Action: config.QuickReplace,
 					Value: []string{
-						"dot://1.1.1.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"dot://cloudflare-dns.com?ip=1.1.1.2&name=Cloudflare&blockedif=zeroip",
-						"dot://1.0.0.2:853?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"dot://cloudflare-dns.com?ip=1.0.0.2&name=Cloudflare&blockedif=zeroip",
 					},
 				},
 				{
 					Name:   "Cloudflare (with Malware Filter) DoH",
 					Action: config.QuickReplace,
 					Value: []string{
-						"doh://1.1.1.2:443?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"https://cloudflare-dns.com/dns-query?ip=1.1.1.2&name=Cloudflare",
-						"doh://1.0.0.2:443?verify=cloudflare-dns.com&name=Cloudflare&blockedif=zeroip",
+						"https://cloudflare-dns.com/dns-query?ip=1.0.0.2&name=Cloudflare",
 					},
 				},
 			},

resolver/resolver-https.go

@@ -12,12 +12,13 @@ import (
 	"github.com/miekg/dns"
 )
 
-// TCPResolver is a resolver using just a single tcp connection with pipelining.
+// HttpsResolver is a resolver using just a single tcp connection with pipelining.
 type HttpsResolver struct {
 	BasicResolverConn
+	Client *http.Client
 }
 
-// tcpQuery holds the query information for a tcpResolverConn.
+// HttpsQuery holds the query information for a httpsResolverConn.
 type HttpsQuery struct {
 	Query    *Query
 	Response chan *dns.Msg
@@ -36,12 +37,23 @@ func (tq *HttpsQuery) MakeCacheRecord(reply *dns.Msg, resolverInfo *ResolverInfo
 	}
 }
 
-// NewTCPResolver returns a new TPCResolver.
+// NewHTTPSResolver returns a new HttpsResolver.
-func NewHttpsResolver(resolver *Resolver) *HttpsResolver {
+func NewHTTPSResolver(resolver *Resolver) *HttpsResolver {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+			ServerName: resolver.VerifyDomain,
+			// TODO: use portbase rng
+		},
+	}
+
+	client := &http.Client{Transport: tr}
+
 	newResolver := &HttpsResolver{
 		BasicResolverConn: BasicResolverConn{
 			resolver: resolver,
 		},
+		Client: client,
 	}
 	newResolver.BasicResolverConn.init()
 	return newResolver
@@ -58,39 +70,23 @@ func (hr *HttpsResolver) Query(ctx context.Context, q *Query) (*RRCache, error)
 	if err != nil {
 		return nil, err
 	}
-
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			MinVersion: tls.VersionTLS12,
-			ServerName: hr.resolver.VerifyDomain,
-			// TODO: use portbase rng
-		},
-	}
-
 	b64dns := base64.RawStdEncoding.EncodeToString(buf)
 
 	url := &url.URL{
 		Scheme:     "https",
 		Host:       hr.resolver.ServerAddress,
-		Path:       fmt.Sprintf("%s/dns-query", hr.resolver.Path), // "dns-query" path is specified in rfc-8484 (https://www.rfc-editor.org/rfc/rfc8484.html)
+		Path:       hr.resolver.Path,
 		ForceQuery: true,
 		RawQuery:   fmt.Sprintf("dns=%s", b64dns),
 	}
 
-	request := &http.Request{
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
-		Method:     "GET",
+
-		URL:        url,
+	if err != nil {
-		Proto:      "HTTP/1.1",
+		return nil, err
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     make(http.Header),
-		Body:       nil,
-		Host:       hr.resolver.ServerAddress,
 	}
 
-	client := &http.Client{Transport: tr}
+	resp, err := hr.Client.Do(request)
-
-	resp, err := client.Do(request)
 
 	if err != nil {
 		return nil, err
@@ -98,8 +94,16 @@ func (hr *HttpsResolver) Query(ctx context.Context, q *Query) (*RRCache, error)
 
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
 	reply := new(dns.Msg)
-	reply.Unpack(body)
+	err = reply.Unpack(body)
+
+	if err != nil {
+		return nil, err
+	}
 
 	newRecord := &RRCache{
 		Domain:   q.FQDN,

resolver/resolver.go

@@ -30,6 +30,10 @@ const (
 	ServerSourceEnv             = "env"
 )
 
+const (
+	HttpsProtocol = "https"
+)
+
 // FailThreshold is amount of errors a resolvers must experience in order to be regarded as failed.
 var FailThreshold = 20
 

resolver/resolvers.go

@@ -1,6 +1,7 @@
 package resolver
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -11,6 +12,7 @@ import (
 
 	"golang.org/x/net/publicsuffix"
 
+	"github.com/miekg/dns"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/utils"
 	"github.com/safing/portmaster/netenv"
@@ -28,6 +30,7 @@ type Scope struct {
 const (
 	parameterName       = "name"
 	parameterVerify     = "verify"
+	parameterIP         = "ip"
 	parameterBlockedIf  = "blockedif"
 	parameterSearch     = "search"
 	parameterSearchOnly = "search-only"
@@ -96,71 +99,55 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 
 	switch u.Scheme {
 	case ServerTypeDNS, ServerTypeDoT, ServerTypeDoH, ServerTypeTCP:
+	case HttpsProtocol:
+		u.Scheme = ServerTypeDoH
 	default:
 		return nil, false, fmt.Errorf("DNS resolver scheme %q invalid", u.Scheme)
 	}
 
 	ip := net.ParseIP(u.Hostname())
-	if ip == nil {
+	isHostnameDomain := (ip == nil)
+	if ip == nil && u.Scheme != ServerTypeDoH {
 		return nil, false, fmt.Errorf("resolver IP %q invalid", u.Hostname())
 	}
 
 	// Add default port for scheme if it is missing.
-	var port uint16
+	port, err := parsePortFromURL(u)
-	hostPort := u.Port()
+	if err != nil {
-	switch {
+		return nil, false, err
-	case hostPort != "":
-		parsedPort, err := strconv.ParseUint(hostPort, 10, 16)
-		if err != nil {
-			return nil, false, fmt.Errorf("resolver port %q invalid", u.Port())
-		}
-		port = uint16(parsedPort)
-	case u.Scheme == ServerTypeDNS, u.Scheme == ServerTypeTCP:
-		port = 53
-	case u.Scheme == ServerTypeDoH:
-		port = 443
-	case u.Scheme == ServerTypeDoT:
-		port = 853
-	default:
-		return nil, false, fmt.Errorf("missing port in %q", u.Host)
-	}
-
-	scope := netutils.GetIPScope(ip)
-	// Skip localhost resolvers from the OS, but not if configured.
-	if scope.IsLocalhost() && source == ServerSourceOperatingSystem {
-		return nil, true, nil // skip
 	}
 
 	// Get parameters and check if keys exist.
 	query := u.Query()
-	for key := range query {
+	err = checkURLParameterValidity(u.Scheme, isHostnameDomain, query)
-		switch key {
+	if err != nil {
-		case parameterName,
+		return nil, false, err
-			parameterVerify,
-			parameterBlockedIf,
-			parameterSearch,
-			parameterSearchOnly,
-			parameterPath:
-			// Known key, continue.
-		default:
-			// Unknown key, abort.
-			return nil, false, fmt.Errorf(`unknown parameter "%s"`, key)
-		}
 	}
 
 	// Check domain verification config.
+	serverAddress := ""
+	paramterServerIP := query.Get(parameterIP)
 	verifyDomain := query.Get(parameterVerify)
-	if verifyDomain != "" && !(u.Scheme == ServerTypeDoT || u.Scheme == ServerTypeDoH) {
+
-		return nil, false, fmt.Errorf("domain verification only supported in DoT and DoH")
+	if u.Scheme == ServerTypeDoT || u.Scheme == ServerTypeDoH {
-	}
+		switch {
-	if verifyDomain == "" && (u.Scheme == ServerTypeDoT || u.Scheme == ServerTypeDoH) {
+		case isHostnameDomain && paramterServerIP != "": // domain and ip as parameter
-		return nil, false, fmt.Errorf("DOT must have a verify query parameter set")
+			ip = net.ParseIP(paramterServerIP)
+			serverAddress = net.JoinHostPort(paramterServerIP, strconv.Itoa(int(port)))
+			verifyDomain = u.Hostname()
+		case !isHostnameDomain && verifyDomain != "": // ip and domain as parameter
+			serverAddress = net.JoinHostPort(ip.String(), strconv.Itoa(int(port)))
+		case isHostnameDomain && verifyDomain == "" && paramterServerIP == "": // only domain
+			verifyDomain = u.Hostname()
+		}
+	} else {
+		serverAddress = net.JoinHostPort(ip.String(), strconv.Itoa(int(port)))
 	}
 
 	// Check path for https (doh) request
 	path := query.Get(parameterPath)
-	if path != "" && u.Scheme != "doh" {
+	if u.Path != "" {
-		return nil, false, fmt.Errorf("path parameter is only supported in DoH")
+		path = u.Path
 	}
 
 	if path != "" && !strings.HasPrefix(path, "/") {
@@ -178,6 +165,16 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 		return nil, false, fmt.Errorf("invalid value for upstream block detection (blockedif=)")
 	}
 
+	// Get ip scope if we have ip
+	scope := netutils.Global
+	if ip != nil {
+		scope = netutils.GetIPScope(ip)
+		// Skip localhost resolvers from the OS, but not if configured.
+		if scope.IsLocalhost() && source == ServerSourceOperatingSystem {
+			return nil, true, nil // skip
+		}
+	}
+
 	// Build resolver.
 	newResolver := &Resolver{
 		ConfigURL: resolverURL,
@@ -189,7 +186,7 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 			IPScope: scope,
 			Port:    port,
 		},
-		ServerAddress:          net.JoinHostPort(ip.String(), strconv.Itoa(int(port))),
+		ServerAddress:          serverAddress,
 		VerifyDomain:           verifyDomain,
 		Path:                   path,
 		UpstreamBlockDetection: blockType,
@@ -215,10 +212,162 @@ func createResolver(resolverURL, source string) (*Resolver, bool, error) {
 		}
 	}
 
+	// Resolve ip if was not specfied by the user
+	err = checkAndResolveServerAddressAndPort(newResolver)
+	if err != nil {
+		return nil, false, err
+	}
+
 	newResolver.Conn = resolverConnFactory(newResolver)
 	return newResolver, false, nil
 }
 
+func checkURLParameterValidity(scheme string, isHostnameDomain bool, query url.Values) error {
+	for key := range query {
+		switch key {
+		case parameterName,
+			parameterVerify,
+			parameterIP,
+			parameterBlockedIf,
+			parameterSearch,
+			parameterSearchOnly,
+			parameterPath:
+			// Known key, continue.
+		default:
+			// Unknown key, abort.
+			return fmt.Errorf(`unknown parameter "%s"`, key)
+		}
+	}
+
+	verifyDomain := query.Get(parameterVerify)
+	paramterServerIP := query.Get(parameterIP)
+
+	if scheme == ServerTypeDoT || scheme == ServerTypeDoH {
+
+		switch {
+		case isHostnameDomain && verifyDomain != "":
+			return fmt.Errorf("cannot have verify parameter with a domain as a hostname")
+		case !isHostnameDomain && verifyDomain == "":
+			return fmt.Errorf("verify paremeter must be set when using ip as domain")
+		case !isHostnameDomain && paramterServerIP != "":
+			return fmt.Errorf("cannot have ip parameter while domain is an ip")
+		}
+	} else {
+		if verifyDomain != "" {
+			return fmt.Errorf("domain verification only supported in DoT and DoH")
+		}
+		if verifyDomain == "" && !isHostnameDomain {
+			return fmt.Errorf("DoT must have a verify query parameter set")
+		}
+	}
+
+	if scheme != ServerTypeDoH {
+		path := query.Get(parameterPath)
+		if path != "" {
+			return fmt.Errorf("path parameter is only supported in DoH")
+		}
+	}
+
+	return nil
+}
+
+func checkAndResolveServerAddressAndPort(resolver *Resolver) error {
+	if resolver.ServerAddress == "" {
+		resolverIps, err := resolveDomainIP(context.Background(), resolver.VerifyDomain)
+		if err != nil {
+			return err
+		}
+
+		if len(resolverIps) == 0 {
+			return fmt.Errorf("no valid IPs resolved for %s", resolver.VerifyDomain)
+		}
+		ip := resolverIps[0]
+		port := int(resolver.Info.Port)
+		resolver.ServerAddress = net.JoinHostPort(ip.String(), strconv.Itoa(port))
+		resolver.Info.IP = ip
+		resolver.Info.IPScope = netutils.GetIPScope(ip)
+	}
+
+	return nil
+}
+
+func resolveDomainIP(ctx context.Context, domain string) ([]net.IP, error) {
+	fqdn := domain
+	if !strings.HasSuffix(fqdn, ".") {
+		fqdn += "."
+	}
+	query := &Query{
+		FQDN:  fqdn,
+		QType: dns.Type(dns.TypeA),
+	}
+
+	for _, resolver := range activeResolvers {
+		rr, err := resolver.Conn.Query(ctx, query)
+		if err != nil {
+			log.Error(err.Error())
+			continue
+		}
+
+		return rr.ExportAllARecords(), nil
+	}
+
+	nameserves := netenv.Nameservers()
+	if len(nameserves) == 0 {
+		return nil, fmt.Errorf("unable to resolve domain %s", domain)
+	}
+
+	client := new(dns.Client)
+
+	message := new(dns.Msg)
+	message.SetQuestion(fqdn, dns.TypeA)
+	message.RecursionDesired = true
+	ip := net.JoinHostPort(nameserves[0].IP.String(), "53")
+
+	reply, _, err := client.Exchange(message, ip)
+
+	if err != nil {
+		return nil, err
+	}
+
+	newRecord := &RRCache{
+		Domain:   query.FQDN,
+		Question: query.QType,
+		RCode:    reply.Rcode,
+		Answer:   reply.Answer,
+		Ns:       reply.Ns,
+		Extra:    reply.Extra,
+	}
+
+	return newRecord.ExportAllARecords(), nil
+}
+
+func parsePortFromURL(url *url.URL) (uint16, error) {
+	var port uint16
+	hostPort := url.Port()
+	if hostPort != "" {
+		// There is a port in the url
+		parsedPort, err := strconv.ParseUint(hostPort, 10, 16)
+		if err != nil {
+			return 0, fmt.Errorf("resolver port %q invalid", url.Port())
+		}
+		port = uint16(parsedPort)
+	} else {
+		// set the default port for the protocol
+		switch {
+		case url.Scheme == ServerTypeDNS, url.Scheme == ServerTypeTCP:
+			port = 53
+		case url.Scheme == ServerTypeDoH:
+			port = 443
+		case url.Scheme == ServerTypeDoT:
+			port = 853
+		default:
+			return 0, fmt.Errorf("cannot determine port for %q", url.Scheme)
+		}
+	}
+
+	return port, nil
+}
+
 func configureSearchDomains(resolver *Resolver, searches []string, hardfail bool) error {
 	resolver.Search = make([]string, 0, len(searches))