Merge pull request #1301 from safing/feature/improve-country-data

Improve country data

broadcasts/data.go

@@ -49,7 +49,7 @@ func collectData() interface{} {
 	if ok && locs.Best().LocationOrNil() != nil {
 		loc := locs.Best()
 		data["Location"] = &Location{
-			Country:        loc.Location.Country.ISOCode,
+			Country:        loc.Location.Country.Code,
 			Coordinates:    loc.Location.Coordinates,
 			ASN:            loc.Location.AutonomousSystemNumber,
 			ASOrg:          loc.Location.AutonomousSystemOrganization,

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/safing/jess v0.3.1
 	github.com/safing/portbase v0.17.2
 	github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626
-	github.com/safing/spn v0.6.16
+	github.com/safing/spn v0.6.17
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.7.0
 	github.com/spkg/zipfs v0.7.1
@@ -95,6 +95,6 @@ require (
 	gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5 // indirect
 	modernc.org/libc v1.24.1 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.7.0 // indirect
+	modernc.org/memory v1.7.1 // indirect
 	modernc.org/sqlite v1.25.0 // indirect
 )

go.sum

@@ -212,8 +212,8 @@ github.com/safing/portbase v0.17.2 h1:HzJkURMmXkv30wMHB7xJ+Z5U5aTMe+EzvlHavKoKko
 github.com/safing/portbase v0.17.2/go.mod h1:1cVgDZIsPiqM5b+K88Kshir5PGIvsftYkx7y1x925+8=
 github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 h1:olc/REnUdpJN/Gmz8B030OxLpMYxyPDTrDILNEw0eKs=
 github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE=
-github.com/safing/spn v0.6.16 h1:z80H3X6wjsmizjWgichqYRyw3hSVB7rJpsDUDL2EWo0=
-github.com/safing/spn v0.6.16/go.mod h1:2CuZfJJazIYyMDrhiwX2eFal0urQyLiX8rXLvJiCTcw=
+github.com/safing/spn v0.6.17 h1:3Lu1cpTcy8zYhA/2UEfeG08Rx1nlwIj1aobSfNXXgUI=
+github.com/safing/spn v0.6.17/go.mod h1:2CuZfJJazIYyMDrhiwX2eFal0urQyLiX8rXLvJiCTcw=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0=
@@ -437,8 +437,8 @@ modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
 modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
 modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/memory v1.7.0 h1:2pXdbgdP5hIyDp2JqIwkHNZ1sAjEbh8GnRpcqFWBf7E=
-modernc.org/memory v1.7.0/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M=
+modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
 modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA=
 modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU=
 zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o=

intel/entity.go

@@ -251,7 +251,7 @@ func (e *Entity) getLocation(ctx context.Context) {
 			return
 		}
 		e.location = loc
-		e.Country = loc.Country.ISOCode
+		e.Country = loc.Country.Code
 		e.Coordinates = &loc.Coordinates
 		e.ASN = loc.AutonomousSystemNumber
 		e.ASOrg = loc.AutonomousSystemOrganization
@@ -272,9 +272,10 @@ func (e *Entity) getLocation(ctx context.Context) {
 
 			// Log location
 			log.Tracer(ctx).Tracef(
-				"intel: located %s in %s (AS%d by %s)%s",
+				"intel: located %s in %s (%s), as part of AS%d by %s%s",
 				e.IP,
-				loc.Country.ISOCode,
+				loc.Country.Name,
+				loc.Country.Code,
 				loc.AutonomousSystemNumber,
 				loc.AutonomousSystemOrganization,
 				flags,
@@ -303,6 +304,16 @@ func (e *Entity) GetCountry(ctx context.Context) (string, bool) {
 	return e.Country, true
 }
 
+// GetCountryInfo returns the two letter ISO country code and whether it is set.
+func (e *Entity) GetCountryInfo(ctx context.Context) *geoip.CountryInfo {
+	e.getLocation(ctx)
+
+	if e.LocationError != "" {
+		return nil
+	}
+	return &e.location.Country
+}
+
 // GetASN returns the AS number and whether it is set.
 func (e *Entity) GetASN(ctx context.Context) (uint, bool) {
 	e.getLocation(ctx)

intel/geoip/country_info_test.go

@@ -1,6 +1,7 @@
 package geoip
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -8,21 +9,31 @@ func TestCountryInfo(t *testing.T) {
 	t.Parallel()
 
 	for key, country := range countries {
-		if key != country.ID {
-			t.Errorf("%s has a wrong ID of %q", key, country.ID)
+		if key != country.Code {
+			t.Errorf("%s has a wrong country code of %q", key, country.Code)
 		}
 		if country.Name == "" {
 			t.Errorf("%s is missing name", key)
 		}
-		if country.Region == "" {
-			t.Errorf("%s is missing region", key)
-		}
-		if country.ContinentCode == "" {
+		if country.Continent.Code == "" {
 			t.Errorf("%s is missing continent", key)
 		}
+		if country.Continent.Region == "" {
+			t.Errorf("%s is missing continent region", key)
+		}
+		if country.Continent.Name == "" {
+			t.Errorf("%s is missing continent name", key)
+		}
+		generatedContinentCode, _, _ := strings.Cut(country.Continent.Region, "-")
+		if country.Continent.Code != generatedContinentCode {
+			t.Errorf("%s is has wrong continent code or region", key)
+		}
 		if country.Center.Latitude == 0 && country.Center.Longitude == 0 {
 			t.Errorf("%s is missing coords", key)
 		}
+		if country.Center.AccuracyRadius == 0 {
+			t.Errorf("%s is missing accuracy radius", key)
+		}
 
 		// Generate map source from data:
 		// fmt.Printf(

intel/geoip/location.go

@@ -18,13 +18,7 @@ const (
 // Location holds information regarding the geographical and network location of an IP address.
 // TODO: We are currently re-using the Continent-Code for the region. Update this and all dependencies.
 type Location struct {
-	Continent struct {
-		Code string `maxminddb:"code"`
-	} `maxminddb:"continent"`
-	Country struct {
-		Name    string
-		ISOCode string `maxminddb:"iso_code"`
-	} `maxminddb:"country"`
+	Country                      CountryInfo `maxminddb:"country"`
 	Coordinates                  Coordinates `maxminddb:"location"`
 	AutonomousSystemNumber       uint        `maxminddb:"autonomous_system_number"`
 	AutonomousSystemOrganization string      `maxminddb:"autonomous_system_organization"`
@@ -96,10 +90,9 @@ const (
 // EstimateNetworkProximity aims to calculate the distance between two network locations. Returns a proximity value between 0 (far away) and 100 (nearby).
 func (l *Location) EstimateNetworkProximity(to *Location) (proximity float32) {
 	switch {
-	case l.Country.ISOCode != "" && l.Country.ISOCode == to.Country.ISOCode:
+	case l.Country.Code != "" && l.Country.Code == to.Country.Code:
 		proximity += weightCountryMatch + weightRegionMatch + weightRegionalNeighborMatch
-	case l.Continent.Code != "" && l.Continent.Code == to.Continent.Code:
-		// FYI: This is the region code!
+	case l.Country.Continent.Region != "" && l.Country.Continent.Region == to.Country.Continent.Region:
 		proximity += weightRegionMatch + weightRegionalNeighborMatch
 	case l.IsRegionalNeighbor(to):
 		proximity += weightRegionalNeighborMatch

intel/geoip/regions.go

@@ -6,11 +6,11 @@ import (
 
 // IsRegionalNeighbor returns whether the supplied location is a regional neighbor.
 func (l *Location) IsRegionalNeighbor(other *Location) bool {
-	if l.Continent.Code == "" || other.Continent.Code == "" {
+	if l.Country.Continent.Region == "" || other.Country.Continent.Region == "" {
 		return false
 	}
-	if region, ok := regions[l.Continent.Code]; ok {
-		return utils.StringInSlice(region.Neighbors, other.Continent.Code)
+	if region, ok := regions[l.Country.Continent.Region]; ok {
+		return utils.StringInSlice(region.Neighbors, other.Country.Continent.Region)
 	}
 	return false
 }

netenv/location.go

@@ -135,12 +135,8 @@ func (dl *DeviceLocation) IsMoreAccurateThan(other *DeviceLocation) bool {
 		other.Location.AutonomousSystemNumber == 0:
 		// Having an ASN is better than having none.
 		return true
-	case dl.Location.Continent.Code != "" &&
-		other.Location.Continent.Code == "":
-		// Having a Continent is better than having none.
-		return true
-	case dl.Location.Country.ISOCode != "" &&
-		other.Location.Country.ISOCode == "":
+	case dl.Location.Country.Code != "" &&
+		other.Location.Country.Code == "":
 		// Having a Country is better than having none.
 		return true
 	case (dl.Location.Coordinates.Latitude != 0 ||
@@ -178,7 +174,13 @@ func (dl *DeviceLocation) String() string {
 			dl.Location.Coordinates.Longitude,
 		)
 	default:
-		return fmt.Sprintf("%s (AS%d in %s)", dl.IP, dl.Location.AutonomousSystemNumber, dl.Location.Country.ISOCode)
+		return fmt.Sprintf(
+			"%s (AS%d in %s - %s)",
+			dl.IP,
+			dl.Location.AutonomousSystemNumber,
+			dl.Location.Country.Name,
+			dl.Location.Country.Code,
+		)
 	}
 }
 
@@ -255,7 +257,7 @@ func (dls *DeviceLocations) AddIP(ip net.IP, source DeviceLocationSource) (dl *D
 		return nil, false
 	}
 	// Only use location if there is data for it.
-	if geoLoc.Country.ISOCode == "" {
+	if geoLoc.Country.Code == "" {
 		return nil, false
 	}
 	loc.Location = geoLoc

profile/endpoints/endpoint-continent.go

@@ -0,0 +1,63 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/safing/portmaster/intel"
+)
+
+var (
+	continentCodePrefix = "C:"
+	continentRegex      = regexp.MustCompile(`^C:[A-Z]{2}$`)
+)
+
+// EndpointContinent matches countries.
+type EndpointContinent struct {
+	EndpointBase
+
+	ContinentCode string
+}
+
+// Matches checks whether the given entity matches this endpoint definition.
+func (ep *EndpointContinent) Matches(ctx context.Context, entity *intel.Entity) (EPResult, Reason) {
+	if entity.IP == nil {
+		return NoMatch, nil
+	}
+
+	if !entity.IPScope.IsGlobal() {
+		return NoMatch, nil
+	}
+
+	countryInfo := entity.GetCountryInfo(ctx)
+	if countryInfo == nil {
+		return MatchError, ep.makeReason(ep, "", "country data not available to match")
+	}
+
+	if ep.ContinentCode == countryInfo.Continent.Code {
+		return ep.match(
+			ep, entity,
+			fmt.Sprintf("%s (%s)", countryInfo.Continent.Name, countryInfo.Continent.Code),
+			"IP is located in",
+		)
+	}
+
+	return NoMatch, nil
+}
+
+func (ep *EndpointContinent) String() string {
+	return ep.renderPPP(continentCodePrefix + ep.ContinentCode)
+}
+
+func parseTypeContinent(fields []string) (Endpoint, error) {
+	if continentRegex.MatchString(fields[1]) {
+		ep := &EndpointContinent{
+			ContinentCode: strings.TrimPrefix(strings.ToUpper(fields[1]), continentCodePrefix),
+		}
+		return ep.parsePPP(ep, fields)
+	}
+
+	return nil, nil
+}

profile/endpoints/endpoint-country.go

@@ -2,6 +2,7 @@ package endpoints
 
 import (
 	"context"
+	"fmt"
 	"regexp"
 	"strings"
 
@@ -14,7 +15,7 @@ var countryRegex = regexp.MustCompile(`^[A-Z]{2}$`)
 type EndpointCountry struct {
 	EndpointBase
 
-	Country string
+	CountryCode string
 }
 
 // Matches checks whether the given entity matches this endpoint definition.
@@ -27,25 +28,29 @@ func (ep *EndpointCountry) Matches(ctx context.Context, entity *intel.Entity) (E
 		return NoMatch, nil
 	}
 
-	country, ok := entity.GetCountry(ctx)
-	if !ok {
-		return MatchError, ep.makeReason(ep, country, "country data not available to match")
+	countryInfo := entity.GetCountryInfo(ctx)
+	if countryInfo == nil {
+		return MatchError, ep.makeReason(ep, "", "country data not available to match")
 	}
 
-	if country == ep.Country {
-		return ep.match(ep, entity, country, "IP is located in")
+	if ep.CountryCode == countryInfo.Code {
+		return ep.match(
+			ep, entity,
+			fmt.Sprintf("%s (%s)", countryInfo.Name, countryInfo.Code),
+			"IP is located in",
+		)
 	}
 	return NoMatch, nil
 }
 
 func (ep *EndpointCountry) String() string {
-	return ep.renderPPP(ep.Country)
+	return ep.renderPPP(ep.CountryCode)
 }
 
 func parseTypeCountry(fields []string) (Endpoint, error) {
 	if countryRegex.MatchString(fields[1]) {
 		ep := &EndpointCountry{
-			Country: strings.ToUpper(fields[1]),
+			CountryCode: strings.ToUpper(fields[1]),
 		}
 		return ep.parsePPP(ep, fields)
 	}

profile/endpoints/endpoint.go

@@ -232,6 +232,10 @@ func parseEndpoint(value string) (endpoint Endpoint, err error) { //nolint:gocog
 	if endpoint, err = parseTypeCountry(fields); endpoint != nil || err != nil {
 		return
 	}
+	// continent
+	if endpoint, err = parseTypeContinent(fields); endpoint != nil || err != nil {
+		return
+	}
 	// asn
 	if endpoint, err = parseTypeASN(fields); endpoint != nil || err != nil {
 		return