wip: migrate to mono-repo. SPN has already been moved to spn/

cmds/observation-hub/.gitignore

@@ -0,0 +1,3 @@
+# Compiled binaries
+observation-hub
+observation-hub.exe

cmds/observation-hub/Dockerfile

@@ -0,0 +1,38 @@
+# Docker Image for Observation Hub
+
+# Important:
+# You need to build this from the repo root!
+# Run: docker build -f cmds/observation-hub/Dockerfile -t safing/observation-hub:latest .
+# Check With: docker run -ti --rm safing/observation-hub:latest --help
+
+# golang 1.21 linux/amd64 on debian bookworm
+# https://github.com/docker-library/golang/blob/master/1.21/bookworm/Dockerfile
+FROM golang:1.21-bookworm as builder
+
+# Ensure ca-certficates are up to date
+RUN update-ca-certificates
+
+# Install dependencies
+WORKDIR $GOPATH/src/github.com/safing/portmaster/spn
+COPY go.mod .
+COPY go.sum .
+ENV GO111MODULE=on
+RUN go mod download
+RUN go mod verify
+
+# Copy source code
+COPY . .
+
+# Build the static binary
+RUN cd cmds/observation-hub && \
+CGO_ENABLED=0 ./build -o /go/bin/observation-hub
+
+# Use static image
+# https://github.com/GoogleContainerTools/distroless
+FROM gcr.io/distroless/static-debian12
+
+# Copy our static executable
+COPY --from=builder --chmod=0755 /go/bin/observation-hub /go/bin/observation-hub
+
+# Run the observation-hub binary.
+ENTRYPOINT ["/go/bin/observation-hub"]

cmds/observation-hub/apprise.go

@@ -0,0 +1,257 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	_ "embed"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/safing/portbase/apprise"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/service/intel/geoip"
+)
+
+var (
+	appriseModule   *modules.Module
+	appriseNotifier *apprise.Notifier
+
+	appriseURL        string
+	appriseTag        string
+	appriseClientCert string
+	appriseClientKey  string
+	appriseGreet      bool
+)
+
+func init() {
+	appriseModule = modules.Register("apprise", nil, startApprise, nil)
+
+	flag.StringVar(&appriseURL, "apprise-url", "", "set the apprise URL to enable notifications via apprise")
+	flag.StringVar(&appriseTag, "apprise-tag", "", "set the apprise tag(s) according to their docs")
+	flag.StringVar(&appriseClientCert, "apprise-client-cert", "", "set the apprise client certificate")
+	flag.StringVar(&appriseClientKey, "apprise-client-key", "", "set the apprise client key")
+	flag.BoolVar(&appriseGreet, "apprise-greet", false, "send a greeting message to apprise on start")
+}
+
+func startApprise() error {
+	// Check if apprise should be configured.
+	if appriseURL == "" {
+		return nil
+	}
+	// Check if there is a tag.
+	if appriseTag == "" {
+		return errors.New("an apprise tag is required")
+	}
+
+	// Create notifier.
+	appriseNotifier = &apprise.Notifier{
+		URL:           appriseURL,
+		DefaultType:   apprise.TypeInfo,
+		DefaultTag:    appriseTag,
+		DefaultFormat: apprise.FormatMarkdown,
+		AllowUntagged: false,
+	}
+
+	if appriseClientCert != "" || appriseClientKey != "" {
+		// Load client cert from disk.
+		cert, err := tls.LoadX509KeyPair(appriseClientCert, appriseClientKey)
+		if err != nil {
+			return fmt.Errorf("failed to load client cert/key: %w", err)
+		}
+
+		// Set client cert in http client.
+		appriseNotifier.SetClient(&http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					MinVersion:   tls.VersionTLS12,
+					Certificates: []tls.Certificate{cert},
+				},
+			},
+			Timeout: 10 * time.Second,
+		})
+	}
+
+	if appriseGreet {
+		err := appriseNotifier.Send(appriseModule.Ctx, &apprise.Message{
+			Title: "👋 Observation Hub Reporting In",
+			Body:  "I am the Observation Hub. I am connected to the SPN and watch out for it. I will report notable changes to the network here.",
+		})
+		if err != nil {
+			log.Warningf("apprise: failed to send test message: %s", err)
+		} else {
+			log.Info("apprise: sent greeting message")
+		}
+	}
+
+	return nil
+}
+
+func reportToApprise(change *observedChange) (errs error) {
+	// Check if configured.
+	if appriseNotifier == nil {
+		return nil
+	}
+
+handleTag:
+	for _, tag := range strings.Split(appriseNotifier.DefaultTag, ",") {
+		// Check if we are shutting down.
+		if appriseModule.IsStopping() {
+			return nil
+		}
+
+		// Render notification based on tag / destination.
+		buf := &bytes.Buffer{}
+		switch {
+		case strings.HasPrefix(tag, "matrix-"):
+			if err := templates.ExecuteTemplate(buf, "matrix-notification", change); err != nil {
+				return fmt.Errorf("failed to render notification: %w", err)
+			}
+
+		case strings.HasPrefix(tag, "discord-"):
+			if err := templates.ExecuteTemplate(buf, "discord-notification", change); err != nil {
+				return fmt.Errorf("failed to render notification: %w", err)
+			}
+
+		default:
+			// Use matrix notification template as default for now.
+			if err := templates.ExecuteTemplate(buf, "matrix-notification", change); err != nil {
+				return fmt.Errorf("failed to render notification: %w", err)
+			}
+		}
+
+		// Send notification to apprise.
+		var err error
+		for i := 0; i < 3; i++ {
+			// Try three times.
+			err = appriseNotifier.Send(appriseModule.Ctx, &apprise.Message{
+				Body: buf.String(),
+				Tag:  tag,
+			})
+			if err == nil {
+				continue handleTag
+			}
+			// Wait for 5 seconds, then try again.
+			time.Sleep(5 * time.Second)
+		}
+		// Add error to errors.
+		if err != nil {
+			errs = errors.Join(errs, fmt.Errorf("| failed to send: %w", err))
+		}
+	}
+
+	return errs
+}
+
+// var (
+// 	entityTemplate = template.Must(template.New("entity").Parse(
+// 		`Entity: {{ . }}
+// {{ .IP }} [{{ .ASN }} - {{ .ASOrg }}]
+// `,
+// 	))
+
+// 	// {{ with .GetCountryInfo -}}
+// 	// {{ .Name }} ({{ .Code }})
+// 	// {{- end }}
+
+// 	matrixTemplate = template.Must(template.New("matrix observer notification").Parse(
+// 		`{{ .Title }}
+// {{ if .Summary }}
+// Details:
+// {{ .Summary }}
+
+// Note: Changes were registered at {{ .UpdateTime }} and were possibly merged.
+// {{ end }}
+
+// {{ template "entity" .UpdatedPin.EntityV4 }}
+
+// Hub Info:
+// Test: {{ .UpdatedPin.EntityV4 }}
+// {{ template "entity" .UpdatedPin.EntityV4 }}
+// {{ template "entity" .UpdatedPin.EntityV6 }}
+// `,
+// 	))
+
+// 	discordTemplate = template.Must(template.New("discord observer notification").Parse(
+// 		``,
+// 	))
+
+// 	defaultTemplate = template.Must(template.New("default observer notification").Parse(
+// 		``,
+// 	))
+// )
+
+var (
+	//go:embed notifications.tmpl
+	templateFile string
+	templates    = template.Must(template.New("notifications").Funcs(
+		template.FuncMap{
+			"joinStrings":    joinStrings,
+			"textBlock":      textBlock,
+			"getCountryInfo": getCountryInfo,
+		},
+	).Parse(templateFile))
+)
+
+func joinStrings(slice []string, sep string) string {
+	return strings.Join(slice, sep)
+}
+
+func textBlock(block, addPrefix, addSuffix string) string {
+	// Trim whitespaces.
+	block = strings.TrimSpace(block)
+
+	// Prepend and append string for every line.
+	lines := strings.Split(block, "\n")
+	for i, line := range lines {
+		lines[i] = addPrefix + line + addSuffix
+	}
+
+	// Return as block.
+	return strings.Join(lines, "\n")
+}
+
+func getCountryInfo(code string) geoip.CountryInfo {
+	// Get the country info directly instead of via the entity location,
+	// so it also works in test without the geoip module.
+	return geoip.GetCountryInfo(code)
+}
+
+// func init() {
+// 	templates = template.Must(template.New(templateFile).Parse(templateFile))
+
+// 	nt, err := templates.New("entity").Parse(
+// 		`Entity: {{ . }}
+// {{ .IP }} [{{ .ASN }} - {{ .ASOrg }}]
+// `,
+// 	)
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// 	templates.AddParseTree(nt.Tree)
+
+// 	if _, err := templates.New("matrix-notification").Parse(
+// 		`{{ .Title }}
+// {{ if .Summary }}
+// Details:
+// {{ .Summary }}
+
+// Note: Changes were registered at {{ .UpdateTime }} and were possibly merged.
+// {{ end }}
+
+// {{ template "entity" .UpdatedPin.EntityV4 }}
+
+// Hub Info:
+// Test: {{ .UpdatedPin.EntityV4 }}
+// {{ template "entity" .UpdatedPin.EntityV4 }}
+// {{ template "entity" .UpdatedPin.EntityV6 }}
+// `,
+// 	); err != nil {
+// 		panic(err)
+// 	}
+// }

cmds/observation-hub/apprise_test.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/safing/portmaster/service/intel"
+	"github.com/safing/portmaster/service/network/netutils"
+	"github.com/safing/portmaster/spn/hub"
+	"github.com/safing/portmaster/spn/navigator"
+)
+
+var observedTestChange = &observedChange{
+	Title: "Hub Changed: fogos (8uLe-zUkC)",
+	Summary: `ConnectedTo.ZwqBAzGqifBAFKFW1GQijNM18pi7BnWH34GyKBF7KB5fC5.HubID removed ZwqBAzGqifBAFKFW1GQijNM18pi7BnWH34GyKBF7KB5fC5
+	ConnectedTo.ZwqBAzGqifBAFKFW1GQijNM18pi7BnWH34GyKBF7KB5fC5.Capacity removed 3403661
+	ConnectedTo.ZwqBAzGqifBAFKFW1GQijNM18pi7BnWH34GyKBF7KB5fC5.Latency removed 252.350006ms`,
+	UpdatedPin: &navigator.PinExport{
+		ID:        "Zwtb8EKMatnMRkW1VaLh8CPV3QswD9iuRU4Sda8uLezUkC",
+		Name:      "fogos",
+		Map:       "main",
+		FirstSeen: time.Now(),
+		EntityV4: &intel.Entity{
+			IP:      net.IPv4(138, 201, 140, 70),
+			IPScope: netutils.Global,
+			Country: "DE",
+			ASN:     24940,
+			ASOrg:   "Hetzner Online GmbH",
+		},
+		States:        []string{"HasRequiredInfo", "Reachable", "Active", "Trusted"},
+		VerifiedOwner: "Safing",
+		HopDistance:   3,
+		SessionActive: false,
+		Info: &hub.Announcement{
+			ID:             "Zwtb8EKMatnMRkW1VaLh8CPV3QswD9iuRU4Sda8uLezUkC",
+			Timestamp:      1677682008,
+			Name:           "fogos",
+			Group:          "Safing",
+			ContactAddress: "abuse@safing.io",
+			ContactService: "email",
+			Hosters:        []string{"Hetzner"},
+			Datacenter:     "DE-Hetzner-FSN",
+			IPv4:           net.IPv4(138, 201, 140, 70),
+			IPv6:           net.ParseIP("2a01:4f8:172:3753::2"),
+			Transports:     []string{"tcp:17", "tcp:17017"},
+			Entry:          []string{},
+			Exit:           []string{"- * TCP/25"},
+		},
+		Status: &hub.Status{
+			Timestamp: 1694180778,
+			Version:   "0.6.19 ",
+		},
+	},
+	UpdateTime: time.Now(),
+}
+
+func TestNotificationTemplate(t *testing.T) {
+	t.Parallel()
+
+	fmt.Println("==========\nFound templates:")
+	for _, tpl := range templates.Templates() {
+		fmt.Println(tpl.Name())
+	}
+	fmt.Println("")
+
+	fmt.Println("\n\n==========\nMatrix template:")
+	matrixOutput := &bytes.Buffer{}
+	err := templates.ExecuteTemplate(matrixOutput, "matrix-notification", observedTestChange)
+	if err != nil {
+		t.Errorf("failed to render matrix template: %s", err)
+	}
+	fmt.Println(matrixOutput.String())
+
+	fmt.Println("\n\n==========\nDiscord template:")
+	discordOutput := &bytes.Buffer{}
+	err = templates.ExecuteTemplate(discordOutput, "discord-notification", observedTestChange)
+	if err != nil {
+		t.Errorf("failed to render discord template: %s", err)
+	}
+	fmt.Println(discordOutput.String())
+}

cmds/observation-hub/build

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# get build data
+if [[ "$BUILD_COMMIT" == "" ]]; then
+  BUILD_COMMIT=$(git describe --all --long --abbrev=99 --dirty 2>/dev/null)
+fi
+if [[ "$BUILD_USER" == "" ]]; then
+  BUILD_USER=$(id -un)
+fi
+if [[ "$BUILD_HOST" == "" ]]; then
+  BUILD_HOST=$(hostname -f)
+fi
+if [[ "$BUILD_DATE" == "" ]]; then
+  BUILD_DATE=$(date +%d.%m.%Y)
+fi
+if [[ "$BUILD_SOURCE" == "" ]]; then
+  BUILD_SOURCE=$(git remote -v | grep origin | cut -f2 | cut -d" " -f1 | head -n 1)
+fi
+if [[ "$BUILD_SOURCE" == "" ]]; then
+  BUILD_SOURCE=$(git remote -v | cut -f2 | cut -d" " -f1 | head -n 1)
+fi
+BUILD_BUILDOPTIONS=$(echo $* | sed "s/ /§/g")
+
+# check
+if [[ "$BUILD_COMMIT" == "" ]]; then
+  echo "could not automatically determine BUILD_COMMIT, please supply manually as environment variable."
+  exit 1
+fi
+if [[ "$BUILD_USER" == "" ]]; then
+  echo "could not automatically determine BUILD_USER, please supply manually as environment variable."
+  exit 1
+fi
+if [[ "$BUILD_HOST" == "" ]]; then
+  echo "could not automatically determine BUILD_HOST, please supply manually as environment variable."
+  exit 1
+fi
+if [[ "$BUILD_DATE" == "" ]]; then
+  echo "could not automatically determine BUILD_DATE, please supply manually as environment variable."
+  exit 1
+fi
+if [[ "$BUILD_SOURCE" == "" ]]; then
+  echo "could not automatically determine BUILD_SOURCE, please supply manually as environment variable."
+  exit 1
+fi
+
+# set build options
+export CGO_ENABLED=0
+if [[ $1 == "dev" ]]; then
+  shift
+  export CGO_ENABLED=1
+  DEV="-race"
+fi
+
+echo "Please notice, that this build script includes metadata into the build."
+echo "This information is useful for debugging and license compliance."
+echo "Run the compiled binary with the -version flag to see the information included."
+
+# build
+BUILD_PATH="github.com/safing/portbase/info"
+go build $DEV -ldflags "-X ${BUILD_PATH}.commit=${BUILD_COMMIT} -X ${BUILD_PATH}.buildOptions=${BUILD_BUILDOPTIONS} -X ${BUILD_PATH}.buildUser=${BUILD_USER} -X ${BUILD_PATH}.buildHost=${BUILD_HOST} -X ${BUILD_PATH}.buildDate=${BUILD_DATE} -X ${BUILD_PATH}.buildSource=${BUILD_SOURCE}" $*

cmds/observation-hub/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/safing/portbase/api"
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/metrics"
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portbase/run"
+	"github.com/safing/portmaster/service/updates"
+	"github.com/safing/portmaster/service/updates/helper"
+	"github.com/safing/portmaster/spn/captain"
+	"github.com/safing/portmaster/spn/conf"
+	"github.com/safing/portmaster/spn/sluice"
+)
+
+func main() {
+	info.Set("SPN Observation Hub", "0.7.1", "AGPLv3", true)
+
+	// Configure metrics.
+	_ = metrics.SetNamespace("observer")
+
+	// Configure user agent.
+	updates.UserAgent = fmt.Sprintf("SPN Observation Hub (%s %s)", runtime.GOOS, runtime.GOARCH)
+	helper.IntelOnly()
+
+	// Configure SPN mode.
+	conf.EnableClient(true)
+	conf.EnablePublicHub(false)
+	captain.DisableAccount = true
+
+	// Disable unneeded listeners.
+	sluice.EnableListener = false
+	api.EnableServer = false
+
+	// Disable module management, as we want to start all modules.
+	modules.DisableModuleManagement()
+
+	// Start.
+	os.Exit(run.Run())
+}

cmds/observation-hub/notifications.tmpl

@@ -0,0 +1,75 @@
+{{ define "entity" -}}
+  {{ .IP }} [AS{{ .ASN }} - {{ .ASOrg }}] in {{ if .Country }}
+    {{- with getCountryInfo .Country -}}
+      {{ .Name }} ({{ .Code }}; Region {{ .Continent.Region }})
+    {{- end }}
+  {{- end }}
+{{- end }}
+
+{{ define "matrix-notification" -}}
+### 🌍 {{ .Title }}{{ if .Summary }}
+
+{{ textBlock .Summary "" "  " }}
+{{ end }}
+
+> Note: Changes were registered at {{ .UpdateTime.UTC.Format "15:04:05 02.01.2006 MST" }} and were possibly merged.
+
+##### Hub Info
+
+> Name: {{ .UpdatedPin.Name }}  
+> ID: {{ .UpdatedPin.ID }}  
+> IPv4: {{ if .UpdatedPin.EntityV4 }}{{ template "entity" .UpdatedPin.EntityV4 }}{{ end }}  
+> IPv6: {{ if .UpdatedPin.EntityV6 }}{{ template "entity" .UpdatedPin.EntityV6 }}{{ end }}  
+> Version: {{ .UpdatedPin.Status.Version }}  
+> States: {{ joinStrings .UpdatedPin.States ", " }}  
+> Status: {{ len .UpdatedPin.Status.Lanes }} Lanes, {{ len .UpdatedPin.Status.Keys }} Keys, {{ .UpdatedPin.Status.Load }} Load  
+> Verified Owner: {{ .UpdatedPin.VerifiedOwner }}  
+> Transports: {{ joinStrings .UpdatedPin.Info.Transports ", " }}  
+> Entry: {{ joinStrings .UpdatedPin.Info.Entry ", " }}  
+> Exit: {{ joinStrings .UpdatedPin.Info.Exit ", " }}  
+> Relations: {{ if .UpdatedPin.Info.Group -}}
+Group={{ .UpdatedPin.Info.Group }} {{ end }}
+
+{{- if .UpdatedPin.Info.Datacenter -}}
+Datacenter={{ .UpdatedPin.Info.Datacenter }} {{ end }}
+
+{{- if .UpdatedPin.Info.Hosters -}}
+Hosters={{ joinStrings .UpdatedPin.Info.Hosters ";" }} {{ end }}
+
+{{- if .UpdatedPin.Info.ContactAddress -}}
+Contact= {{ .UpdatedPin.Info.ContactAddress }}{{ if .UpdatedPin.Info.ContactService }} via {{ .UpdatedPin.Info.ContactService }}{{ end }}{{ end }}
+
+{{- end }}
+
+{{ define "discord-notification" -}}
+# 🌍 {{ .Title }}{{ if .Summary }}
+
+{{ .Summary }}
+{{- end }}
+
+##### Note: Changes were registered at {{ .UpdateTime.UTC.Format "15:04:05 02.01.2006 MST" }} and were possibly merged. - Hub Info:
+
+Name: {{ .UpdatedPin.Name }}
+ID: {{ .UpdatedPin.ID }}
+IPv4: {{ if .UpdatedPin.EntityV4 }}{{ template "entity" .UpdatedPin.EntityV4 }}{{ end }}
+IPv6: {{ if .UpdatedPin.EntityV6 }}{{ template "entity" .UpdatedPin.EntityV6 }}{{ end }}
+Version: {{ .UpdatedPin.Status.Version }}
+States: {{ joinStrings .UpdatedPin.States ", " }}
+Status: {{ len .UpdatedPin.Status.Lanes }} Lanes, {{ len .UpdatedPin.Status.Keys }} Keys, {{ .UpdatedPin.Status.Load }} Load
+Verified Owner: {{ .UpdatedPin.VerifiedOwner }}
+Transports: {{ joinStrings .UpdatedPin.Info.Transports ", " }}
+Entry: {{ joinStrings .UpdatedPin.Info.Entry ", " }}
+Exit: {{ joinStrings .UpdatedPin.Info.Exit ", " }}
+Relations: {{ if .UpdatedPin.Info.Group -}}
+Group={{ .UpdatedPin.Info.Group }} {{ end }}
+
+{{- if .UpdatedPin.Info.Datacenter -}}
+Datacenter={{ .UpdatedPin.Info.Datacenter }} {{ end }}
+
+{{- if .UpdatedPin.Info.Hosters -}}
+Hosters={{ joinStrings .UpdatedPin.Info.Hosters ";" }} {{ end }}
+
+{{- if .UpdatedPin.Info.ContactAddress -}}
+Contact= {{ .UpdatedPin.Info.ContactAddress }}{{ if .UpdatedPin.Info.ContactService }} via {{ .UpdatedPin.Info.ContactService }}{{ end }}{{ end }}
+
+{{- end }}

cmds/observation-hub/observe.go

@@ -0,0 +1,407 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	diff "github.com/r3labs/diff/v3"
+	"golang.org/x/exp/slices"
+
+	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/database/query"
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/modules"
+	"github.com/safing/portmaster/spn/captain"
+	"github.com/safing/portmaster/spn/navigator"
+)
+
+var (
+	observerModule *modules.Module
+
+	db = database.NewInterface(&database.Options{
+		Local:    true,
+		Internal: true,
+	})
+
+	reportAllChanges bool
+
+	errNoChanges = errors.New("no changes")
+
+	reportingDelayFlag string
+	reportingDelay     = 10 * time.Minute
+)
+
+func init() {
+	observerModule = modules.Register("observer", prepObserver, startObserver, nil, "captain", "apprise")
+
+	flag.BoolVar(&reportAllChanges, "report-all-changes", false, "report all changes, no just interesting ones")
+	flag.StringVar(&reportingDelayFlag, "reporting-delay", "10m", "delay reports to summarize changes")
+}
+
+func prepObserver() error {
+	if reportingDelayFlag != "" {
+		duration, err := time.ParseDuration(reportingDelayFlag)
+		if err != nil {
+			return fmt.Errorf("failed to parse reporting-delay: %w", err)
+		}
+		reportingDelay = duration
+	}
+
+	return nil
+}
+
+func startObserver() error {
+	observerModule.StartServiceWorker("observer", 0, observerWorker)
+
+	return nil
+}
+
+type observedPin struct {
+	previous *navigator.PinExport
+	latest   *navigator.PinExport
+
+	lastUpdate         time.Time
+	lastUpdateReported bool
+}
+
+type observedChange struct {
+	Title   string
+	Summary string
+
+	UpdatedPin *navigator.PinExport
+	UpdateTime time.Time
+
+	SPNStatus *captain.SPNStatus
+}
+
+func observerWorker(ctx context.Context) error {
+	log.Info("observer: starting")
+	defer log.Info("observer: stopped")
+
+	// Subscribe to SPN status.
+	statusSub, err := db.Subscribe(query.New("runtime:spn/status"))
+	if err != nil {
+		return fmt.Errorf("failed to subscribe to spn status: %w", err)
+	}
+	defer statusSub.Cancel() //nolint:errcheck
+
+	// Get latest status.
+	latestStatus := captain.GetSPNStatus()
+
+	// Step 1: Wait for SPN to connect, if needed.
+	if latestStatus.Status != captain.StatusConnected {
+		log.Info("observer: waiting for SPN to connect")
+	waitForConnect:
+		for {
+			select {
+			case r := <-statusSub.Feed:
+				if r == nil {
+					return errors.New("status feed ended")
+				}
+
+				statusUpdate, ok := r.(*captain.SPNStatus)
+				switch {
+				case !ok:
+					log.Warningf("observer: received invalid SPN status: %s", r)
+				case statusUpdate.Status == captain.StatusFailed:
+					log.Warningf("observer: SPN failed to connect")
+				case statusUpdate.Status == captain.StatusConnected:
+					break waitForConnect
+				}
+			case <-ctx.Done():
+				return nil
+			}
+		}
+	}
+
+	// Wait for one second for the navigator to settle things.
+	log.Info("observer: connected to network, waiting for navigator")
+	time.Sleep(1 * time.Second)
+
+	// Step 2: Get current state.
+	mapQuery := query.New("map:main/")
+	q, err := db.Query(mapQuery)
+	if err != nil {
+		return fmt.Errorf("failed to start map query: %w", err)
+	}
+	defer q.Cancel()
+
+	// Put all current pins in a map.
+	observedPins := make(map[string]*observedPin)
+query:
+	for {
+		select {
+		case r := <-q.Next:
+			// Check if we are done.
+			if r == nil {
+				break query
+			}
+			// Add all pins to seen pins.
+			if pin, ok := r.(*navigator.PinExport); ok {
+				observedPins[pin.ID] = &observedPin{
+					previous: pin,
+					latest:   pin,
+				}
+			} else {
+				log.Warningf("observer: received invalid pin export: %s", r)
+			}
+		case <-ctx.Done():
+			return nil
+		}
+	}
+	if q.Err() != nil {
+		return fmt.Errorf("failed to finish map query: %w", q.Err())
+	}
+
+	// Step 3: Monitor for changes.
+	sub, err := db.Subscribe(mapQuery)
+	if err != nil {
+		return fmt.Errorf("failed to start map sub: %w", err)
+	}
+	defer sub.Cancel() //nolint:errcheck
+
+	// Start ticker for checking for changes.
+	reportChangesTicker := time.NewTicker(10 * time.Second)
+	defer reportChangesTicker.Stop()
+
+	log.Info("observer: listening for hub changes")
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+
+		case r := <-statusSub.Feed:
+			// Keep SPN connection status up to date.
+			if r == nil {
+				return errors.New("status feed ended")
+			}
+			if statusUpdate, ok := r.(*captain.SPNStatus); ok {
+				latestStatus = statusUpdate
+				log.Infof("observer: SPN status is now %s", statusUpdate.Status)
+			} else {
+				log.Warningf("observer: received invalid pin export: %s", r)
+			}
+
+		case r := <-sub.Feed:
+			// Save all observed pins.
+			switch {
+			case r == nil:
+				return errors.New("pin feed ended")
+			case r.Meta().IsDeleted():
+				delete(observedPins, path.Base(r.DatabaseKey()))
+			default:
+				if pin, ok := r.(*navigator.PinExport); ok {
+					existingObservedPin, ok := observedPins[pin.ID]
+					if ok {
+						// Update previously observed Hub.
+						existingObservedPin.latest = pin
+						existingObservedPin.lastUpdate = time.Now()
+						existingObservedPin.lastUpdateReported = false
+					} else {
+						// Add new Hub.
+						observedPins[pin.ID] = &observedPin{
+							latest:     pin,
+							lastUpdate: time.Now(),
+						}
+					}
+				} else {
+					log.Warningf("observer: received invalid pin export: %s", r)
+				}
+			}
+
+		case <-reportChangesTicker.C:
+			// Report changed pins.
+
+			for _, observedPin := range observedPins {
+				// Check if context was canceled.
+				select {
+				case <-ctx.Done():
+					return nil
+				default:
+				}
+
+				switch {
+				case observedPin.lastUpdateReported:
+					// Change already reported.
+				case time.Since(observedPin.lastUpdate) < reportingDelay:
+					// Only report changes if older than the configured delay.
+				default:
+					// Format and report.
+					title, changes, err := formatPinChanges(observedPin.previous, observedPin.latest)
+					if err != nil {
+						if !errors.Is(err, errNoChanges) {
+							log.Warningf("observer: failed to format pin changes: %s", err)
+						}
+					} else {
+						// Report changes.
+						reportChanges(&observedChange{
+							Title:      title,
+							Summary:    changes,
+							UpdatedPin: observedPin.latest,
+							UpdateTime: observedPin.lastUpdate,
+							SPNStatus:  latestStatus,
+						})
+					}
+
+					// Update observed pin.
+					observedPin.previous = observedPin.latest
+					observedPin.lastUpdateReported = true
+				}
+			}
+		}
+	}
+}
+
+func reportChanges(change *observedChange) {
+	// Log changes.
+	log.Infof("observer:\n%s\n%s", change.Title, change.Summary)
+
+	// Report via Apprise.
+	err := reportToApprise(change)
+	if err != nil {
+		log.Warningf("observer: failed to report changes to apprise: %s", err)
+	}
+}
+
+var (
+	ignoreChangesIn = []string{
+		"ConnectedTo",
+		"HopDistance",
+		"Info.entryPolicy", // Alternatively, ignore "Info.Entry"
+		"Info.exitPolicy",  // Alternatively, ignore "Info.Exit"
+		"Info.parsedTransports",
+		"Info.Timestamp",
+		"SessionActive",
+		"Status.Keys",
+		"Status.Lanes",
+		"Status.Load",
+		"Status.Timestamp",
+	}
+
+	ignoreStates = []string{
+		"IsHomeHub",
+		"Failing",
+	}
+)
+
+func ignoreChange(path string) bool {
+	if reportAllChanges {
+		return false
+	}
+
+	for _, pathPrefix := range ignoreChangesIn {
+		if strings.HasPrefix(path, pathPrefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func formatPinChanges(from, to *navigator.PinExport) (title, changes string, err error) {
+	// Return immediately if pin is new.
+	if from == nil {
+		return fmt.Sprintf("New Hub: %s", makeHubName(to.Name, to.ID)), "", nil
+	}
+
+	// Find notable changes.
+	changelog, err := diff.Diff(from, to)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to diff: %w", err)
+	}
+	if len(changelog) > 0 {
+		// Build changelog message.
+		changes := make([]string, 0, len(changelog))
+		for _, change := range changelog {
+			// Create path to changed field.
+			fullPath := strings.Join(change.Path, ".")
+
+			// Check if this path should be ignored.
+			if ignoreChange(fullPath) {
+				continue
+			}
+
+			// Add to reportable changes.
+			changeMsg := formatChange(change, fullPath)
+			if changeMsg != "" {
+				changes = append(changes, changeMsg)
+			}
+		}
+
+		// Log the changes, if there are any left.
+		if len(changes) > 0 {
+			return fmt.Sprintf("Hub Changed: %s", makeHubName(to.Name, to.ID)),
+				strings.Join(changes, "\n"),
+				nil
+		}
+	}
+
+	return "", "", errNoChanges
+}
+
+func formatChange(change diff.Change, fullPath string) string {
+	switch {
+	case strings.HasPrefix(fullPath, "States"):
+		switch change.Type {
+		case diff.CREATE:
+			return formatState(fmt.Sprintf("%v", change.To), true)
+		case diff.UPDATE:
+			a := formatState(fmt.Sprintf("%v", change.To), true)
+			b := formatState(fmt.Sprintf("%v", change.From), false)
+			switch {
+			case a != "" && b != "":
+				return a + "\n" + b
+			case a != "":
+				return a
+			case b != "":
+				return b
+			}
+		case diff.DELETE:
+			return formatState(fmt.Sprintf("%v", change.From), false)
+		}
+
+	default:
+		switch change.Type {
+		case diff.CREATE:
+			return fmt.Sprintf("%s added %v", fullPath, change.To)
+		case diff.UPDATE:
+			return fmt.Sprintf("%s changed from %v to %v", fullPath, change.From, change.To)
+		case diff.DELETE:
+			return fmt.Sprintf("%s removed %v", fullPath, change.From)
+		}
+	}
+
+	return ""
+}
+
+func formatState(name string, isSet bool) string {
+	// Check if state should be ignored.
+	if !reportAllChanges && slices.Contains[[]string, string](ignoreStates, name) {
+		return ""
+	}
+
+	if isSet {
+		return fmt.Sprintf("State is %v", name)
+	}
+	return fmt.Sprintf("State is NOT %v", name)
+}
+
+func makeHubName(name, id string) string {
+	shortenedID := id[len(id)-8:len(id)-4] +
+		"-" +
+		id[len(id)-4:]
+
+	// Be more careful, as the Hub name is user input.
+	switch {
+	case name == "":
+		return shortenedID
+	case len(name) > 16:
+		return fmt.Sprintf("%s (%s)", name[:16], shortenedID)
+	default:
+		return fmt.Sprintf("%s (%s)", name, shortenedID)
+	}
+}