Move and fix geoip package

intel/geoip/database.go

@@ -0,0 +1,103 @@
+package geoip
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/tevino/abool"
+
+	maxminddb "github.com/oschwald/maxminddb-golang"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/updater"
+	"github.com/safing/portmaster/updates"
+)
+
+var (
+	dbCityFile *updater.File
+	dbASNFile  *updater.File
+	dbFileLock sync.Mutex
+
+	dbCity *maxminddb.Reader
+	dbASN  *maxminddb.Reader
+	dbLock sync.Mutex
+
+	dbInUse    = abool.NewBool(false) // only activate if used for first time
+	dbDoReload = abool.NewBool(true)  // if database should be reloaded
+)
+
+// ReloadDatabases reloads the geoip database, if they are in use.
+func ReloadDatabases() error {
+	// don't do anything if the database isn't actually used
+	if !dbInUse.IsSet() {
+		return nil
+	}
+
+	dbFileLock.Lock()
+	defer dbFileLock.Unlock()
+	dbLock.Lock()
+	defer dbLock.Unlock()
+
+	dbDoReload.Set()
+	return doReload()
+}
+
+func prepDatabaseForUse() error {
+	dbInUse.Set()
+	return doReload()
+}
+
+func doReload() error {
+	// reload if needed
+	if dbDoReload.SetToIf(true, false) {
+		closeDBs()
+		return openDBs()
+	}
+
+	return nil
+}
+
+func openDBs() error {
+	var err error
+	file, err := updates.GetFile("intel/geoip/geoip-city.mmdb")
+	if err != nil {
+		return fmt.Errorf("could not get GeoIP City database file: %s", err)
+	}
+	dbCity, err = maxminddb.Open(file.Path())
+	if err != nil {
+		return err
+	}
+
+	file, err = updates.GetFile("intel/geoip/geoip-asn.mmdb")
+	if err != nil {
+		return fmt.Errorf("could not get GeoIP ASN database file: %s", err)
+	}
+	dbASN, err = maxminddb.Open(file.Path())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func handleError(err error) {
+	log.Errorf("network/geoip: lookup failed, reloading databases: %s", err)
+	dbDoReload.Set()
+}
+
+func closeDBs() {
+	if dbCity != nil {
+		err := dbCity.Close()
+		if err != nil {
+			log.Warningf("network/geoip: failed to close database: %s", err)
+		}
+	}
+	dbCity = nil
+
+	if dbASN != nil {
+		err := dbASN.Close()
+		if err != nil {
+			log.Warningf("network/geoip: failed to close database: %s", err)
+		}
+	}
+	dbASN = nil
+}

intel/geoip/location.go

@@ -0,0 +1,136 @@
+package geoip
+
+import (
+	"encoding/binary"
+	"net"
+
+	"github.com/umahmood/haversine"
+)
+
+const (
+	earthCircumferenceInKm float64 = 40100 // earth circumference in km
+)
+
+// Location holds information regarding the geographical and network location of an IP address
+type Location struct {
+	Continent struct {
+		Code string `maxminddb:"code"`
+	} `maxminddb:"continent"`
+	Country struct {
+		ISOCode string `maxminddb:"iso_code"`
+	} `maxminddb:"country"`
+	Coordinates struct {
+		AccuracyRadius uint16  `maxminddb:"accuracy_radius"`
+		Latitude       float64 `maxminddb:"latitude"`
+		Longitude      float64 `maxminddb:"longitude"`
+	} `maxminddb:"location"`
+	AutonomousSystemNumber       uint   `maxminddb:"autonomous_system_number"`
+	AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
+}
+
+// About GeoLite2 City accuracy_radius:
+//
+// range: 1-1000
+// seen values (from memory): 1,5,10,20,50,100,200,500,1000
+// default seems to be 100
+//
+// examples:
+// 1.1.1/24 has 1000: Anycast
+// 8.8.0/19 has 1000: Anycast
+// 8.8.52/22 has 1: City of Westfield
+//
+// Conclusion:
+// - Ignore location data completely if accuracy_radius > 500
+
+// EstimateNetworkProximity aims to calculate the distance between two network locations. Returns a proximity value between 0 (far away) and 100 (nearby).
+func (l *Location) EstimateNetworkProximity(to *Location) (proximity int) {
+	// Distance Value:
+	// 0: other side of the Internet
+	// 100: same network/datacenter
+
+	// Weighting:
+	// coordinate distance: 0-50
+	// continent match: 15
+	// country match: 10
+	// AS owner match: 15
+	// AS network match: 10
+
+	// coordinate distance: 0-50
+	fromCoords := haversine.Coord{Lat: l.Coordinates.Latitude, Lon: l.Coordinates.Longitude}
+	toCoords := haversine.Coord{Lat: to.Coordinates.Latitude, Lon: to.Coordinates.Longitude}
+	_, km := haversine.Distance(fromCoords, toCoords)
+
+	// proximity distance by accuracy
+	// get worst accuracy rating
+	accuracy := l.Coordinates.AccuracyRadius
+	if to.Coordinates.AccuracyRadius > accuracy {
+		accuracy = to.Coordinates.AccuracyRadius
+	}
+
+	if km <= 10 && accuracy <= 100 {
+		proximity += 50
+	} else {
+		distanceIn50Percent := ((earthCircumferenceInKm - km) / earthCircumferenceInKm) * 50
+
+		// apply penalty for locations with low accuracy (targeting accuracy radius >100)
+		accuracyModifier := 1 - float64(accuracy)/1000
+		proximity += int(distanceIn50Percent * accuracyModifier)
+	}
+
+	// continent match: 15
+	if l.Continent.Code == to.Continent.Code {
+		proximity += 15
+		// country match: 10
+		if l.Country.ISOCode == to.Country.ISOCode {
+			proximity += 10
+		}
+	}
+
+	// AS owner match: 15
+	if l.AutonomousSystemOrganization == to.AutonomousSystemOrganization {
+		proximity += 15
+		// AS network match: 10
+		if l.AutonomousSystemNumber == to.AutonomousSystemNumber {
+			proximity += 10
+		}
+	}
+
+	return //nolint:nakedret
+}
+
+// PrimitiveNetworkProximity calculates the numerical distance between two IP addresses. Returns a proximity value between 0 (far away) and 100 (nearby).
+func PrimitiveNetworkProximity(from net.IP, to net.IP, ipVersion uint8) int {
+
+	var diff float64
+
+	switch ipVersion {
+	case 4:
+		a := binary.BigEndian.Uint32(from[12:])
+		b := binary.BigEndian.Uint32(to[12:])
+		if a > b {
+			diff = float64(a - b)
+		} else {
+			diff = float64(b - a)
+		}
+	case 6:
+		a := binary.BigEndian.Uint64(from[:8])
+		b := binary.BigEndian.Uint64(to[:8])
+		if a > b {
+			diff = float64(a - b)
+		} else {
+			diff = float64(b - a)
+		}
+	default:
+		return 0
+	}
+
+	switch ipVersion {
+	case 4:
+		diff /= 256
+		return int((1 - diff/16777216) * 100)
+	case 6:
+		return int((1 - diff/18446744073709552000) * 100)
+	default:
+		return 0
+	}
+}

intel/geoip/location_test.go

@@ -0,0 +1,40 @@
+package geoip
+
+import (
+	"net"
+	"testing"
+)
+
+func TestPrimitiveNetworkProximity(t *testing.T) {
+	ip4_1 := net.ParseIP("1.1.1.1")
+	ip4_2 := net.ParseIP("1.1.1.2")
+	ip4_3 := net.ParseIP("255.255.255.0")
+
+	dist := PrimitiveNetworkProximity(ip4_1, ip4_2, 4)
+	t.Logf("primitive proximity %s <> %s: %d", ip4_1, ip4_2, dist)
+	if dist < 90 {
+		t.Fatalf("unexpected distance between ip4_1 and ip4_2: %d", dist)
+	}
+
+	dist = PrimitiveNetworkProximity(ip4_1, ip4_3, 4)
+	t.Logf("primitive proximity %s <> %s: %d", ip4_1, ip4_3, dist)
+	if dist > 10 {
+		t.Fatalf("unexpected distance between ip4_1 and ip4_3: %d", dist)
+	}
+
+	ip6_1 := net.ParseIP("2a02::1")
+	ip6_2 := net.ParseIP("2a02::2")
+	ip6_3 := net.ParseIP("ffff::1")
+
+	dist = PrimitiveNetworkProximity(ip6_1, ip6_2, 6)
+	t.Logf("primitive proximity %s <> %s: %d", ip6_1, ip6_2, dist)
+	if dist < 90 {
+		t.Fatalf("unexpected distance between ip6_1 and ip6_2: %d", dist)
+	}
+
+	dist = PrimitiveNetworkProximity(ip6_1, ip6_3, 6)
+	t.Logf("primitive proximity %s <> %s: %d", ip6_1, ip6_3, dist)
+	if dist > 20 {
+		t.Fatalf("unexpected distance between ip6_1 and ip6_3: %d", dist)
+	}
+}

intel/geoip/lookup.go

@@ -0,0 +1,46 @@
+package geoip
+
+import (
+	"net"
+)
+
+// GetLocation returns Location data of an IP address
+func GetLocation(ip net.IP) (record *Location, err error) {
+	dbLock.Lock()
+	defer dbLock.Unlock()
+
+	err = prepDatabaseForUse()
+	if err != nil {
+		return nil, err
+	}
+
+	record = &Location{}
+
+	// fetch
+	err = dbCity.Lookup(ip, record)
+	if err == nil {
+		err = dbASN.Lookup(ip, record)
+	}
+
+	// retry
+	if err != nil {
+		// reprep
+		handleError(err)
+		err = prepDatabaseForUse()
+		if err != nil {
+			return nil, err
+		}
+
+		// refetch
+		err = dbCity.Lookup(ip, record)
+		if err == nil {
+			err = dbASN.Lookup(ip, record)
+		}
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return record, nil
+}

intel/geoip/lookup_test.go

@@ -0,0 +1,60 @@
+package geoip
+
+import (
+	"net"
+	"testing"
+)
+
+func TestLocationLookup(t *testing.T) {
+	ip1 := net.ParseIP("81.2.69.142")
+	loc1, err := GetLocation(ip1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc1)
+
+	ip2 := net.ParseIP("1.1.1.1")
+	loc2, err := GetLocation(ip2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc2)
+
+	ip3 := net.ParseIP("8.8.8.8")
+	loc3, err := GetLocation(ip3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc3)
+
+	ip4 := net.ParseIP("81.2.70.142")
+	loc4, err := GetLocation(ip4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc4)
+
+	ip5 := net.ParseIP("194.232.1.1")
+	loc5, err := GetLocation(ip5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc5)
+
+	ip6 := net.ParseIP("151.101.1.164")
+	loc6, err := GetLocation(ip6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc6)
+
+	dist1 := loc1.EstimateNetworkProximity(loc2)
+	dist2 := loc2.EstimateNetworkProximity(loc3)
+	dist3 := loc1.EstimateNetworkProximity(loc3)
+	dist4 := loc1.EstimateNetworkProximity(loc4)
+
+	t.Logf("proximity %s <> %s: %d", ip1, ip2, dist1)
+	t.Logf("proximity %s <> %s: %d", ip2, ip3, dist2)
+	t.Logf("proximity %s <> %s: %d", ip1, ip3, dist3)
+	t.Logf("proximity %s <> %s: %d", ip1, ip4, dist4)
+}

intel/geoip/module.go

@@ -0,0 +1,41 @@
+package geoip
+
+import (
+	"context"
+
+	"github.com/safing/portbase/modules"
+)
+
+var (
+	module *modules.Module
+)
+
+func init() {
+	module = modules.Register("geoip", prep, nil, nil, "updates")
+}
+
+func prep() error {
+	return module.RegisterEventHook(
+		"updates",
+		"resource update",
+		"upgrade databases",
+		upgradeDatabases,
+	)
+}
+
+func upgradeDatabases(_ context.Context, _ interface{}) error {
+	dbFileLock.Lock()
+	reload := false
+	if dbCityFile != nil && dbCityFile.UpgradeAvailable() {
+		reload = true
+	}
+	if dbASNFile != nil && dbASNFile.UpgradeAvailable() {
+		reload = true
+	}
+	dbFileLock.Unlock()
+
+	if reload {
+		return ReloadDatabases()
+	}
+	return nil
+}

intel/geoip/module_test.go

@@ -0,0 +1,11 @@
+package geoip
+
+import (
+	"testing"
+
+	"github.com/safing/portmaster/core/pmtesting"
+)
+
+func TestMain(m *testing.M) {
+	pmtesting.TestMain(m)
+}