Initial commit after restructure

network/geoip/database.go

@@ -0,0 +1,99 @@
+package geoip
+
+import (
+	"errors"
+	"sync"
+
+	maxminddb "github.com/oschwald/maxminddb-golang"
+
+	"github.com/Safing/safing-core/log"
+	"github.com/Safing/safing-core/update"
+)
+
+var (
+	dbCity *maxminddb.Reader
+	dbASN  *maxminddb.Reader
+
+	dbLock     sync.Mutex
+	dbInUse    = false // only activate if used for first time
+	dbDoReload = true  // if database should be reloaded
+
+	// mmdbCityFile = "/opt/safing/GeoLite2-City.mmdb"
+	// mmdbASNFile  = "/opt/safing/GeoLite2-ASN.mmdb"
+)
+
+func ReloadDatabases() error {
+	dbLock.Lock()
+	defer dbLock.Unlock()
+
+	// don't do anything if the database isn't actually used
+	if !dbInUse {
+		return nil
+	}
+
+	dbDoReload = true
+	return doReload()
+}
+
+func prepDatabaseForUse() error {
+	dbInUse = true
+	return doReload()
+}
+
+func doReload() error {
+	// reload if needed
+	if dbDoReload {
+		defer func() {
+			dbDoReload = false
+		}()
+
+		closeDBs()
+		return openDBs()
+	}
+
+	return nil
+}
+
+func openDBs() error {
+	var err error
+	filepath := update.GetGeoIPCityPath()
+	if filepath == "" {
+		return errors.New("could not get GeoIP City filepath")
+	}
+	dbCity, err = maxminddb.Open(filepath)
+	if err != nil {
+		return err
+	}
+	filepath = update.GetGeoIPASNPath()
+	if filepath == "" {
+		return errors.New("could not get GeoIP ASN filepath")
+	}
+	dbASN, err = maxminddb.Open(filepath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func handleError(err error) {
+	log.Warningf("network/geoip: lookup failed, reloading databases...")
+	dbDoReload = true
+}
+
+func closeDBs() {
+	if dbCity != nil {
+		err := dbCity.Close()
+		if err != nil {
+			log.Warningf("network/geoip: failed to close database: %s", err)
+		}
+	}
+	dbCity = nil
+
+	if dbASN != nil {
+		err := dbASN.Close()
+		if err != nil {
+			log.Warningf("network/geoip: failed to close database: %s", err)
+		}
+	}
+	dbASN = nil
+}

network/geoip/location.go

@@ -0,0 +1,138 @@
+package geoip
+
+import (
+	"encoding/binary"
+	"net"
+
+	"github.com/umahmood/haversine"
+)
+
+const (
+	earthCircumferenceKm float64 = 40100 // earth circumference in km
+)
+
+// Location holds information regarding the geographical and network location of an IP address
+type Location struct {
+	Continent struct {
+		Code string `maxminddb:"code"`
+	} `maxminddb:"continent"`
+	Country struct {
+		ISOCode string `maxminddb:"iso_code"`
+	} `maxminddb:"country"`
+	Coordinates struct {
+		AccuracyRadius uint16  `maxminddb:"accuracy_radius"`
+		Latitude       float64 `maxminddb:"latitude"`
+		Longitude      float64 `maxminddb:"longitude"`
+	} `maxminddb:"location"`
+	AutonomousSystemNumber       uint   `maxminddb:"autonomous_system_number"`
+	AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
+}
+
+// About GeoLite2 City accuracy_radius:
+//
+// range: 1-1000
+// seen values (from memory): 1,5,10,20,50,100,200,500,1000
+// default seems to be 100
+//
+// examples:
+// 1.1.1/24 has 1000: Anycast
+// 8.8.0/19 has 1000: Anycast
+// 8.8.52/22 has 1: City of Westfield
+//
+// Conclusion:
+// - Ignore location data completely if accuracy_radius > 500
+
+// EstimateNetworkProximity aims to calculate a distance value between 0 and 100.
+func (l *Location) EstimateNetworkProximity(to *Location) (proximity int) {
+	// Distance Value:
+	// 0: other side of the Internet
+	// 100: same network/datacenter
+
+	// Weighting:
+	// coordinate distance: 0-50
+	// continent match: 10
+	// country match: 10
+	// AS owner match: 15
+	// AS network match: 15
+	//
+	// We prioritize AS information over country information, as it is more accurate and we expect better privacy if we already are in the destination AS.
+
+	// coordinate distance: 0-50
+	fromCoords := haversine.Coord{Lat: l.Coordinates.Latitude, Lon: l.Coordinates.Longitude}
+	toCoords := haversine.Coord{Lat: to.Coordinates.Latitude, Lon: to.Coordinates.Longitude}
+	_, km := haversine.Distance(fromCoords, toCoords)
+
+	// proximity distance by accuracy
+	// get worst accuracy rating
+	accuracy := l.Coordinates.AccuracyRadius
+	if to.Coordinates.AccuracyRadius > accuracy {
+		accuracy = to.Coordinates.AccuracyRadius
+	}
+
+	if km <= 10 && accuracy <= 200 {
+		proximity += 50
+	} else {
+		distanceIn50Percent := ((earthCircumferenceKm - km) / earthCircumferenceKm) * 50
+
+		// apply penalty for values high values (targeting >100)
+		accuracyModifier := 1 - float64(accuracy)/1000
+		proximity += int(distanceIn50Percent * accuracyModifier)
+	}
+
+	// continent match: 10
+	if l.Continent.Code == to.Continent.Code {
+		proximity += 10
+		// country match: 10
+		if l.Country.ISOCode == to.Country.ISOCode {
+			proximity += 10
+		}
+	}
+
+	// AS owner match: 15
+	if l.AutonomousSystemOrganization == to.AutonomousSystemOrganization {
+		proximity += 15
+		// AS network match: 15
+		if l.AutonomousSystemNumber == to.AutonomousSystemNumber {
+			proximity += 15
+		}
+	}
+
+	return
+
+}
+
+func PrimitiveNetworkProximity(from net.IP, to net.IP, ipVersion uint8) int {
+
+	var diff float64
+
+	switch ipVersion {
+	case 4:
+		a := binary.BigEndian.Uint32(from[12:])
+		b := binary.BigEndian.Uint32(to[12:])
+		if a > b {
+			diff = float64(a - b)
+		} else {
+			diff = float64(b - a)
+		}
+	case 6:
+		a := binary.BigEndian.Uint64(from[:8])
+		b := binary.BigEndian.Uint64(to[:8])
+		if a > b {
+			diff = float64(a - b)
+		} else {
+			diff = float64(b - a)
+		}
+	default:
+		return 0
+	}
+
+	switch ipVersion {
+	case 4:
+		diff = diff / 256
+		return int((1 - diff/16777216) * 100)
+	case 6:
+		return int((1 - diff/18446744073709552000) * 100)
+	default:
+		return 0
+	}
+}

network/geoip/location_test.go

@@ -0,0 +1,40 @@
+package geoip
+
+import (
+	"net"
+	"testing"
+)
+
+func TestPrimitiveNetworkProximity(t *testing.T) {
+	ip4_1 := net.ParseIP("1.1.1.1")
+	ip4_2 := net.ParseIP("1.1.1.2")
+	ip4_3 := net.ParseIP("255.255.255.0")
+
+	dist := PrimitiveNetworkProximity(ip4_1, ip4_2, 4)
+	t.Logf("primitive proximity %s <> %s: %d", ip4_1, ip4_2, dist)
+	if dist < 90 {
+		t.Fatalf("unexpected distance between ip4_1 and ip4_2: %d", dist)
+	}
+
+	dist = PrimitiveNetworkProximity(ip4_1, ip4_3, 4)
+	t.Logf("primitive proximity %s <> %s: %d", ip4_1, ip4_3, dist)
+	if dist > 10 {
+		t.Fatalf("unexpected distance between ip4_1 and ip4_3: %d", dist)
+	}
+
+	ip6_1 := net.ParseIP("2a02::1")
+	ip6_2 := net.ParseIP("2a02::2")
+	ip6_3 := net.ParseIP("ffff::1")
+
+	dist = PrimitiveNetworkProximity(ip6_1, ip6_2, 6)
+	t.Logf("primitive proximity %s <> %s: %d", ip6_1, ip6_2, dist)
+	if dist < 90 {
+		t.Fatalf("unexpected distance between ip6_1 and ip6_2: %d", dist)
+	}
+
+	dist = PrimitiveNetworkProximity(ip6_1, ip6_3, 6)
+	t.Logf("primitive proximity %s <> %s: %d", ip6_1, ip6_3, dist)
+	if dist > 20 {
+		t.Fatalf("unexpected distance between ip6_1 and ip6_3: %d", dist)
+	}
+}

network/geoip/lookup.go

@@ -0,0 +1,45 @@
+package geoip
+
+import (
+	"net"
+)
+
+// GetLocation returns Location data of an IP address
+func GetLocation(ip net.IP) (record *Location, err error) {
+	dbLock.Lock()
+	defer dbLock.Unlock()
+
+	err = prepDatabaseForUse()
+	if err != nil {
+		return nil, err
+	}
+
+	record = &Location{}
+
+	// fetch
+	err = dbCity.Lookup(ip, record)
+	if err == nil {
+		err = dbASN.Lookup(ip, record)
+	}
+
+	// retry
+	if err != nil {
+		// reprep
+		handleError(err)
+		err = prepDatabaseForUse()
+		if err != nil {
+			return nil, err
+		}
+
+		// refetch
+		err = dbCity.Lookup(ip, record)
+		if err == nil {
+			err = dbASN.Lookup(ip, record)
+		}
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	return record, nil
+}

network/geoip/lookup_test.go

@@ -0,0 +1,47 @@
+package geoip
+
+import (
+	"net"
+	"testing"
+)
+
+func TestLocationLookup(t *testing.T) {
+	ip1 := net.ParseIP("81.2.69.142")
+	loc1, err := GetLocation(ip1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc1)
+
+	ip2 := net.ParseIP("1.1.1.1")
+	loc2, err := GetLocation(ip2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc2)
+
+	ip3 := net.ParseIP("8.8.8.8")
+	loc3, err := GetLocation(ip3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc3)
+
+	ip4 := net.ParseIP("81.2.70.142")
+	loc4, err := GetLocation(ip4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", loc1)
+
+	dist1 := loc1.EstimateNetworkProximity(loc2)
+	dist2 := loc2.EstimateNetworkProximity(loc3)
+	dist3 := loc1.EstimateNetworkProximity(loc3)
+	dist4 := loc1.EstimateNetworkProximity(loc4)
+
+	t.Logf("proximity %s <> %s: %d", ip1, ip2, dist1)
+	t.Logf("proximity %s <> %s: %d", ip2, ip3, dist2)
+	t.Logf("proximity %s <> %s: %d", ip1, ip3, dist3)
+	t.Logf("proximity %s <> %s: %d", ip1, ip4, dist4)
+
+}