From d8c25ecb5f0a9d976fae54f726b4c9131dcb4277 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 23 Mar 2023 13:57:32 +0100 Subject: [PATCH] Add geoip regions to improve distance estimation --- intel/geoip/location.go | 24 +- intel/geoip/lookup.go | 1 + intel/geoip/regions.go | 503 ++++++++++++++++++++++++++++++++++++ intel/geoip/regions_test.go | 27 ++ 4 files changed, 545 insertions(+), 10 deletions(-) create mode 100644 intel/geoip/regions.go create mode 100644 intel/geoip/regions_test.go diff --git a/intel/geoip/location.go b/intel/geoip/location.go index afd1911e..1a58077f 100644 --- a/intel/geoip/location.go +++ b/intel/geoip/location.go @@ -16,6 +16,7 @@ const ( ) // Location holds information regarding the geographical and network location of an IP address. +// TODO: We are currently re-using the Continent-Code for the region. Update this and and all dependencies. type Location struct { Continent struct { Code string `maxminddb:"code"` @@ -63,10 +64,13 @@ type Coordinates struct { */ const ( - weightContinentMatch = 20 - weightCountryMatch = 15 - weightASOrgMatch = 10 - weightASNMatch = 5 + weightCountryMatch = 10 + weightRegionMatch = 10 + weightRegionalNeighborMatch = 10 + + weightASNMatch = 10 + weightASOrgMatch = 10 + weightCoordinateDistance = 50 ) @@ -92,12 +96,12 @@ const ( func (l *Location) EstimateNetworkProximity(to *Location) (proximity float32) { switch { case l.Country.ISOCode != "" && l.Country.ISOCode == to.Country.ISOCode: - // Rely more on the Country Code data, as it is more accurate than the - // Continent Code, especially when combining location data from multiple - // sources. - proximity += weightContinentMatch + weightCountryMatch + proximity += weightCountryMatch + weightRegionMatch + weightRegionalNeighborMatch case l.Continent.Code != "" && l.Continent.Code == to.Continent.Code: - proximity += weightContinentMatch + // FYI: This is the region code! + proximity += weightRegionMatch + weightRegionalNeighborMatch + case l.IsRegionalNeighbor(to): + proximity += weightRegionalNeighborMatch } switch { @@ -105,7 +109,7 @@ func (l *Location) EstimateNetworkProximity(to *Location) (proximity float32) { l.AutonomousSystemNumber != 0: // Rely more on the ASN data, as it is more accurate than the ASOrg data, // especially when combining location data from multiple sources. - proximity += weightASOrgMatch + weightASNMatch + proximity += weightASNMatch + weightASOrgMatch case l.AutonomousSystemOrganization == to.AutonomousSystemOrganization && l.AutonomousSystemNumber != 0 && // Check if an ASN is set. If the ASOrg is known, the ASN must be too. !ASOrgUnknown(l.AutonomousSystemOrganization): // Check if the ASOrg name is valid. diff --git a/intel/geoip/lookup.go b/intel/geoip/lookup.go index c635b574..d6f76f87 100644 --- a/intel/geoip/lookup.go +++ b/intel/geoip/lookup.go @@ -24,6 +24,7 @@ func GetLocation(ip net.IP) (*Location, error) { } record.FillMissingInfo() + record.AddRegion() return record, nil } diff --git a/intel/geoip/regions.go b/intel/geoip/regions.go new file mode 100644 index 00000000..0ce18761 --- /dev/null +++ b/intel/geoip/regions.go @@ -0,0 +1,503 @@ +package geoip + +import ( + "github.com/safing/portbase/utils" +) + +// AddRegion adds the region based on the country. +func (l *Location) AddRegion() { + if regionID, ok := countryRegions[l.Country.ISOCode]; ok { + l.Continent.Code = regionID + } +} + +// IsRegionalNeighbor returns whether the supplied location is a regional neighbor. +func (l *Location) IsRegionalNeighbor(other *Location) bool { + if l.Continent.Code == "" || other.Continent.Code == "" { + return false + } + if region, ok := regions[l.Continent.Code]; ok { + return utils.StringInSlice(region.Neighbors, other.Continent.Code) + } + return false +} + +type Region struct { + ID string + Name string + Neighbors []string +} + +var regions = map[string]*Region{ + "AF-C": { + ID: "AF-C", + Name: "Africa, Sub-Saharan Africa, Middle Africa", + Neighbors: []string{ + "AF-E", + "AF-N", + "AF-S", + "AF-W", + }, + }, + "AF-E": { + ID: "AF-E", + Name: "Africa, Sub-Saharan Africa, Eastern Africa", + Neighbors: []string{ + "AF-C", + "AF-N", + "AF-S", + }, + }, + "AF-N": { + ID: "AF-N", + Name: "Africa, Northern Africa", + Neighbors: []string{ + "AF-C", + "AF-E", + "AF-W", + "AS-W", + "EU-S", + }, + }, + "AF-S": { + ID: "AF-S", + Name: "Africa, Sub-Saharan Africa, Southern Africa", + Neighbors: []string{ + "AF-C", + "AF-E", + "AF-W", + }, + }, + "AF-W": { + ID: "AF-W", + Name: "Africa, Sub-Saharan Africa, Western Africa", + Neighbors: []string{ + "AF-C", + "AF-N", + "AF-S", + }, + }, + "AN": { + ID: "AN", + Name: "Antarctica", + Neighbors: []string{}, + }, + "AS-C": { + ID: "AS-C", + Name: "Asia, Central Asia", + Neighbors: []string{ + "AS-E", + "AS-S", + "AS-SE", + "AS-W", + }, + }, + "AS-E": { + ID: "AS-E", + Name: "Asia, Eastern Asia", + Neighbors: []string{ + "AS-C", + "AS-S", + "AS-SE", + }, + }, + "AS-S": { + ID: "AS-S", + Name: "Asia, Southern Asia", + Neighbors: []string{ + "AS-C", + "AS-E", + "AS-SE", + "AS-W", + }, + }, + "AS-SE": { + ID: "AS-SE", + Name: "Asia, South-eastern Asia", + Neighbors: []string{ + "AS-C", + "AS-E", + "AS-S", + "OC-C", + "OC-E", + "OC-N", + "OC-S", + }, + }, + "AS-W": { + ID: "AS-W", + Name: "Asia, Western Asia", + Neighbors: []string{ + "AF-N", + "AS-C", + "AS-S", + "EU-E", + }, + }, + "EU-E": { + ID: "EU-E", + Name: "Europe, Eastern Europe", + Neighbors: []string{ + "AS-W", + "EU-N", + "EU-S", + "EU-W", + }, + }, + "EU-N": { + ID: "EU-N", + Name: "Europe, Northern Europe", + Neighbors: []string{ + "EU-E", + "EU-S", + "EU-W", + }, + }, + "EU-S": { + ID: "EU-S", + Name: "Europe, Southern Europe", + Neighbors: []string{ + "AF-N", + "EU-E", + "EU-N", + "EU-W", + }, + }, + "EU-W": { + ID: "EU-W", + Name: "Europe, Western Europe", + Neighbors: []string{ + "EU-E", + "EU-N", + "EU-S", + }, + }, + "NA-E": { + ID: "NA-E", + Name: "North America, Caribbean", + Neighbors: []string{ + "NA-N", + "NA-S", + "SA", + }, + }, + "NA-N": { + ID: "NA-N", + Name: "North America, Northern America", + Neighbors: []string{ + "NA-E", + "NA-N", + "NA-S", + }, + }, + "NA-S": { + ID: "NA-S", + Name: "North America, Central America", + Neighbors: []string{ + "NA-E", + "NA-N", + "NA-S", + "SA", + }, + }, + "OC-C": { + ID: "OC-C", + Name: "Oceania, Melanesia", + Neighbors: []string{ + "AS-SE", + "OC-E", + "OC-N", + "OC-S", + }, + }, + "OC-E": { + ID: "OC-E", + Name: "Oceania, Polynesia", + Neighbors: []string{ + "AS-SE", + "OC-C", + "OC-N", + "OC-S", + }, + }, + "OC-N": { + ID: "OC-N", + Name: "Oceania, Micronesia", + Neighbors: []string{ + "AS-SE", + "OC-C", + "OC-E", + "OC-S", + }, + }, + "OC-S": { + ID: "OC-S", + Name: "Oceania, Australia and New Zealand", + Neighbors: []string{ + "AS-SE", + "OC-C", + "OC-E", + "OC-N", + }, + }, + "SA": { // TODO: Split up + ID: "SA", + Name: "South America", + Neighbors: []string{ + "NA-E", + "NA-S", + }, + }, +} + +var countryRegions = map[string]string{ + "AF": "AS-S", + "AX": "EU-N", + "AL": "EU-S", + "DZ": "AF-N", + "AS": "OC-E", + "AD": "EU-S", + "AO": "AF-C", + "AI": "NA-E", + "AQ": "AN", + "AG": "NA-E", + "AR": "SA", + "AM": "AS-W", + "AW": "NA-E", + "AU": "OC-S", + "AT": "EU-W", + "AZ": "AS-W", + "BS": "NA-E", + "BH": "AS-W", + "BD": "AS-S", + "BB": "NA-E", + "BY": "EU-E", + "BE": "EU-W", + "BZ": "NA-S", + "BJ": "AF-W", + "BM": "NA-N", + "BT": "AS-S", + "BO": "SA", + "BQ": "NA-E", + "BA": "EU-S", + "BW": "AF-S", + "BV": "SA", + "BR": "SA", + "IO": "AF-E", + "BN": "AS-SE", + "BG": "EU-E", + "BF": "AF-W", + "BI": "AF-E", + "CV": "AF-W", + "KH": "AS-SE", + "CM": "AF-C", + "CA": "NA-N", + "KY": "NA-E", + "CF": "AF-C", + "TD": "AF-C", + "CL": "SA", + "CN": "AS-E", + "CX": "OC-S", + "CC": "OC-S", + "CO": "SA", + "KM": "AF-E", + "CG": "AF-C", + "CD": "AF-C", + "CK": "OC-E", + "CR": "NA-S", + "CI": "AF-W", + "HR": "EU-S", + "CU": "NA-E", + "CW": "NA-E", + "CY": "AS-W", + "CZ": "EU-E", + "DK": "EU-N", + "DJ": "AF-E", + "DM": "NA-E", + "DO": "NA-E", + "EC": "SA", + "EG": "AF-N", + "SV": "NA-S", + "GQ": "AF-C", + "ER": "AF-E", + "EE": "EU-N", + "SZ": "AF-S", + "ET": "AF-E", + "FK": "SA", + "FO": "EU-N", + "FJ": "OC-C", + "FI": "EU-N", + "FR": "EU-W", + "GF": "SA", + "PF": "OC-E", + "TF": "AF-E", + "GA": "AF-C", + "GM": "AF-W", + "GE": "AS-W", + "DE": "EU-W", + "GH": "AF-W", + "GI": "EU-S", + "GR": "EU-S", + "GL": "NA-N", + "GD": "NA-E", + "GP": "NA-E", + "GU": "OC-N", + "GT": "NA-S", + "GG": "EU-N", + "GN": "AF-W", + "GW": "AF-W", + "GY": "SA", + "HT": "NA-E", + "HM": "OC-S", + "VA": "EU-S", + "HN": "NA-S", + "HK": "AS-E", + "HU": "EU-E", + "IS": "EU-N", + "IN": "AS-S", + "ID": "AS-SE", + "IR": "AS-S", + "IQ": "AS-W", + "IE": "EU-N", + "IM": "EU-N", + "IL": "AS-W", + "IT": "EU-S", + "JM": "NA-E", + "JP": "AS-E", + "JE": "EU-N", + "JO": "AS-W", + "KZ": "AS-C", + "KE": "AF-E", + "KI": "OC-N", + "KP": "AS-E", + "KR": "AS-E", + "KW": "AS-W", + "KG": "AS-C", + "LA": "AS-SE", + "LV": "EU-N", + "LB": "AS-W", + "LS": "AF-S", + "LR": "AF-W", + "LY": "AF-N", + "LI": "EU-W", + "LT": "EU-N", + "LU": "EU-W", + "MO": "AS-E", + "MG": "AF-E", + "MW": "AF-E", + "MY": "AS-SE", + "MV": "AS-S", + "ML": "AF-W", + "MT": "EU-S", + "MH": "OC-N", + "MQ": "NA-E", + "MR": "AF-W", + "MU": "AF-E", + "YT": "AF-E", + "MX": "NA-S", + "FM": "OC-N", + "MD": "EU-E", + "MC": "EU-W", + "MN": "AS-E", + "ME": "EU-S", + "MS": "NA-E", + "MA": "AF-N", + "MZ": "AF-E", + "MM": "AS-SE", + "NA": "AF-S", + "NR": "OC-N", + "NP": "AS-S", + "NL": "EU-W", + "NC": "OC-C", + "NZ": "OC-S", + "NI": "NA-S", + "NE": "AF-W", + "NG": "AF-W", + "NU": "OC-E", + "NF": "OC-S", + "MK": "EU-S", + "MP": "OC-N", + "NO": "EU-N", + "OM": "AS-W", + "PK": "AS-S", + "PW": "OC-N", + "PS": "AS-W", + "PA": "NA-S", + "PG": "OC-C", + "PY": "SA", + "PE": "SA", + "PH": "AS-SE", + "PN": "OC-E", + "PL": "EU-E", + "PT": "EU-S", + "PR": "NA-E", + "QA": "AS-W", + "RE": "AF-E", + "RO": "EU-E", + "RU": "EU-E", + "RW": "AF-E", + "BL": "NA-E", + "SH": "AF-W", + "KN": "NA-E", + "LC": "NA-E", + "MF": "NA-E", + "PM": "NA-N", + "VC": "NA-E", + "WS": "OC-E", + "SM": "EU-S", + "ST": "AF-C", + "SA": "AS-W", + "SN": "AF-W", + "RS": "EU-S", + "SC": "AF-E", + "SL": "AF-W", + "SG": "AS-SE", + "SX": "NA-E", + "SK": "EU-E", + "SI": "EU-S", + "SB": "OC-C", + "SO": "AF-E", + "ZA": "AF-S", + "GS": "SA", + "SS": "AF-E", + "ES": "EU-S", + "LK": "AS-S", + "SD": "AF-N", + "SR": "SA", + "SJ": "EU-N", + "SE": "EU-N", + "CH": "EU-W", + "SY": "AS-W", + "TW": "AS-E", + "TJ": "AS-C", + "TZ": "AF-E", + "TH": "AS-SE", + "TL": "AS-SE", + "TG": "AF-W", + "TK": "OC-E", + "TO": "OC-E", + "TT": "NA-E", + "TN": "AF-N", + "TR": "AS-W", + "TM": "AS-C", + "TC": "NA-E", + "TV": "OC-E", + "UG": "AF-E", + "UA": "EU-E", + "AE": "AS-W", + "GB": "EU-N", + "US": "NA-N", + "UM": "OC-N", + "UY": "SA", + "UZ": "AS-C", + "VU": "OC-C", + "VE": "SA", + "VN": "AS-SE", + "VG": "NA-E", + "VI": "NA-E", + "WF": "OC-E", + "EH": "AF-N", + "YE": "AS-W", + "ZM": "AF-E", + "ZW": "AF-E", +} diff --git a/intel/geoip/regions_test.go b/intel/geoip/regions_test.go new file mode 100644 index 00000000..6ea7ae34 --- /dev/null +++ b/intel/geoip/regions_test.go @@ -0,0 +1,27 @@ +package geoip + +import ( + "testing" + + "github.com/safing/portbase/utils" +) + +func TestRegions(t *testing.T) { + t.Parallel() + + // Check if all neighbors are also linked back. + for key, region := range regions { + if key != region.ID { + t.Errorf("region has different key than ID: %s != %s", key, region.ID) + } + for _, neighborID := range region.Neighbors { + if otherRegion, ok := regions[neighborID]; ok { + if !utils.StringInSlice(otherRegion.Neighbors, region.ID) { + t.Errorf("region %s has neighbor %s, but is not linked back", region.ID, neighborID) + } + } else { + t.Errorf("region %s does not exist", neighborID) + } + } + } +}