wip: migrate to mono-repo. SPN has already been moved to spn/
This commit is contained in:
Patrick Pacher
311
spn/patrol/domains.go
Normal file
311
spn/patrol/domains.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package patrol
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
)
|
||||
|
||||
// getRandomTestDomain returns a random test domain from the test domain list.
|
||||
// Not cryptographically secure random, though.
|
||||
func getRandomTestDomain() string {
|
||||
rng := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
|
||||
return testDomains[rng.Intn(len(testDomains)-1)] //nolint:gosec // Weak randomness is not an issue here.
|
||||
}
|
||||
|
||||
// testDomains is a list of domains to check if they respond successfully to a HTTP GET request.
|
||||
// They are sourced from tranco - trimmed, checked, and cleaned.
|
||||
// Use TestCleanDomains to clean a new/updated list.
|
||||
// Treat as a constant.
|
||||
var testDomains = []string{
|
||||
"about.com",
|
||||
"addtoany.com",
|
||||
"adobe.com",
|
||||
"aliyun.com",
|
||||
"ampproject.org",
|
||||
"android.com",
|
||||
"apache.org",
|
||||
"apple.com",
|
||||
"apple.news",
|
||||
"appspot.com",
|
||||
"arnebrachhold.de",
|
||||
"avast.com",
|
||||
"bbc.co.uk",
|
||||
"bbc.com",
|
||||
"bing.com",
|
||||
"blogger.com",
|
||||
"blogspot.com",
|
||||
"branch.io",
|
||||
"calendly.com",
|
||||
"cam.ac.uk",
|
||||
"canonical.com",
|
||||
"canva.com",
|
||||
"cisco.com",
|
||||
"cloudflare.com",
|
||||
"cloudns.net",
|
||||
"cnblogs.com",
|
||||
"cnn.com",
|
||||
"creativecommons.org",
|
||||
"criteo.com",
|
||||
"cupfox.app",
|
||||
"dailymail.co.uk",
|
||||
"ddnss.de",
|
||||
"debian.org",
|
||||
"digitalocean.com",
|
||||
"doi.org",
|
||||
"domainmarket.com",
|
||||
"doubleclick.net",
|
||||
"dreamhost.com",
|
||||
"dropbox.com",
|
||||
"dynect.net",
|
||||
"ed.gov",
|
||||
"elegantthemes.com",
|
||||
"elpais.com",
|
||||
"epa.gov",
|
||||
"eporner.com",
|
||||
"espn.com",
|
||||
"europa.eu",
|
||||
"example.com",
|
||||
"facebook.com",
|
||||
"fb.com",
|
||||
"fb.me",
|
||||
"fb.watch",
|
||||
"fbcdn.net",
|
||||
"feedburner.com",
|
||||
"free.fr",
|
||||
"ftc.gov",
|
||||
"g.page",
|
||||
"getbootstrap.com",
|
||||
"gitlab.com",
|
||||
"gmail.com",
|
||||
"gnu.org",
|
||||
"goo.gl",
|
||||
"google-analytics.com",
|
||||
"google.ca",
|
||||
"google.co.in",
|
||||
"google.co.jp",
|
||||
"google.co.th",
|
||||
"google.co.uk",
|
||||
"google.com.au",
|
||||
"google.com.br",
|
||||
"google.com.hk",
|
||||
"google.com.mx",
|
||||
"google.com.tr",
|
||||
"google.com.tw",
|
||||
"google.com",
|
||||
"google.de",
|
||||
"google.es",
|
||||
"google.fr",
|
||||
"google.it",
|
||||
"googledomains.com",
|
||||
"googlesyndication.com",
|
||||
"gstatic.com",
|
||||
"harvard.edu",
|
||||
"hitomi.la",
|
||||
"hubspot.com",
|
||||
"hugedomains.com",
|
||||
"ibm.com",
|
||||
"icloud.com",
|
||||
"ikea.com",
|
||||
"ilovepdf.com",
|
||||
"indiatimes.com",
|
||||
"instagram.com",
|
||||
"investing.com",
|
||||
"investopedia.com",
|
||||
"irs.gov",
|
||||
"kickstarter.com",
|
||||
"launchpad.net",
|
||||
"lencr.org",
|
||||
"lijit.com",
|
||||
"linkedin.com",
|
||||
"linode.com",
|
||||
"mashable.com",
|
||||
"medium.com",
|
||||
"mega.co.nz",
|
||||
"mega.nz",
|
||||
"merriam-webster.com",
|
||||
"mit.edu",
|
||||
"netflix.com",
|
||||
"nginx.org",
|
||||
"nist.gov",
|
||||
"notion.so",
|
||||
"nsone.net",
|
||||
"office.com",
|
||||
"onetrust.com",
|
||||
"openstreetmap.org",
|
||||
"patreon.com",
|
||||
"pexels.com",
|
||||
"photobucket.com",
|
||||
"php.net",
|
||||
"pki.goog",
|
||||
"plos.org",
|
||||
"ps.kz",
|
||||
"readthedocs.io",
|
||||
"redd.it",
|
||||
"reddit.com",
|
||||
"remove.bg",
|
||||
"rfc-editor.org",
|
||||
"savefrom.net",
|
||||
"sedo.com",
|
||||
"so-net.ne.jp",
|
||||
"sourceforge.net",
|
||||
"spamhaus.org",
|
||||
"speedtest.net",
|
||||
"spotify.com",
|
||||
"stanford.edu",
|
||||
"state.gov",
|
||||
"substack.com",
|
||||
"t.me",
|
||||
"taboola.com",
|
||||
"techcrunch.com",
|
||||
"telegram.me",
|
||||
"telegram.org",
|
||||
"threema.ch",
|
||||
"tinyurl.com",
|
||||
"ubuntu.com",
|
||||
"ui.com",
|
||||
"umich.edu",
|
||||
"uol.com.br",
|
||||
"upenn.edu",
|
||||
"usgs.gov",
|
||||
"utexas.edu",
|
||||
"va.gov",
|
||||
"verisign.com",
|
||||
"vmware.com",
|
||||
"w3.org",
|
||||
"wa.me",
|
||||
"webs.com",
|
||||
"whatsapp.com",
|
||||
"whatsapp.net",
|
||||
"whitehouse.gov",
|
||||
"wikimedia.org",
|
||||
"wikipedia.org",
|
||||
"wiktionary.org",
|
||||
"www.aliyundrive.com",
|
||||
"www.amazon.ca",
|
||||
"www.amazon.co.jp",
|
||||
"www.amazon.co.uk",
|
||||
"www.amazon.com",
|
||||
"www.amazon.de",
|
||||
"www.amazon.es",
|
||||
"www.amazon.fr",
|
||||
"www.amazon.in",
|
||||
"www.amazon.it",
|
||||
"www.aol.com",
|
||||
"www.appsflyer.com",
|
||||
"www.att.com",
|
||||
"www.business.site",
|
||||
"www.ca.gov",
|
||||
"www.canada.ca",
|
||||
"www.cctv.com",
|
||||
"www.cdc.gov",
|
||||
"www.chinaz.com",
|
||||
"www.cloud.com",
|
||||
"www.cnet.com",
|
||||
"www.comcast.com",
|
||||
"www.comcast.net",
|
||||
"www.cornell.edu",
|
||||
"www.crashlytics.com",
|
||||
"www.datadoghq.com",
|
||||
"www.db.com",
|
||||
"www.deloitte.com",
|
||||
"www.dw.com",
|
||||
"www.engadget.com",
|
||||
"www.eset.com",
|
||||
"www.fao.org",
|
||||
"www.fedex.com",
|
||||
"www.flickr.com",
|
||||
"www.force.com",
|
||||
"www.ford.com",
|
||||
"www.frontiersin.org",
|
||||
"www.geeksforgeeks.org",
|
||||
"www.gene.com",
|
||||
"www.genius.com",
|
||||
"www.github.io",
|
||||
"www.gov.uk",
|
||||
"www.gravatar.com",
|
||||
"www.healthline.com",
|
||||
"www.hhs.gov",
|
||||
"www.hichina.com",
|
||||
"www.hinet.net",
|
||||
"www.house.gov",
|
||||
"www.hp.com",
|
||||
"www.huawei.com",
|
||||
"www.hupu.com",
|
||||
"www.ietf.org",
|
||||
"www.immunet.com",
|
||||
"www.independent.co.uk",
|
||||
"www.intel.com",
|
||||
"www.jotform.com",
|
||||
"www.klaviyo.com",
|
||||
"www.launchdarkly.com",
|
||||
"www.live.com",
|
||||
"www.macromedia.com",
|
||||
"www.medallia.com",
|
||||
"www.mediatek.com",
|
||||
"www.medicalnewstoday.com",
|
||||
"www.microsoft.com",
|
||||
"www.mongodb.com",
|
||||
"www.mysql.com",
|
||||
"www.namu.wiki",
|
||||
"www.nasa.gov",
|
||||
"www.nba.com",
|
||||
"www.nbcnews.com",
|
||||
"www.nih.gov",
|
||||
"www.noaa.gov",
|
||||
"www.npr.org",
|
||||
"www.nps.gov",
|
||||
"www.ny.gov",
|
||||
"www.okta.com",
|
||||
"www.openai.com",
|
||||
"www.optimizely.com",
|
||||
"www.oracle.com",
|
||||
"www.outlook.com",
|
||||
"www.paloaltonetworks.com",
|
||||
"www.pbs.org",
|
||||
"www.pixabay.com",
|
||||
"www.plala.or.jp",
|
||||
"www.playstation.com",
|
||||
"www.plesk.com",
|
||||
"www.princeton.edu",
|
||||
"www.prnewswire.com",
|
||||
"www.psu.edu",
|
||||
"www.python.org",
|
||||
"www.qq.com",
|
||||
"www.quantserve.com",
|
||||
"www.quillbot.com",
|
||||
"www.rackspace.com",
|
||||
"www.redhat.com",
|
||||
"www.researchgate.net",
|
||||
"www.roku.com",
|
||||
"www.salesforce.com",
|
||||
"www.skype.com",
|
||||
"www.sun.com",
|
||||
"www.teamviewer.com",
|
||||
"www.ted.com",
|
||||
"www.tesla.com",
|
||||
"www.theguardian.com",
|
||||
"www.typeform.com",
|
||||
"www.uchicago.edu",
|
||||
"www.ucla.edu",
|
||||
"www.usda.gov",
|
||||
"www.usps.com",
|
||||
"www.utorrent.com",
|
||||
"www.warnerbros.com",
|
||||
"www.webex.com",
|
||||
"www.who.int",
|
||||
"www.worldbank.org",
|
||||
"www.xbox.com",
|
||||
"www.xerox.com",
|
||||
"www.youdao.com",
|
||||
"www.zdnet.com",
|
||||
"www.zebra.com",
|
||||
"yahoo.com",
|
||||
"yale.edu",
|
||||
"yandex.com",
|
||||
"yandex.net",
|
||||
"youku.com",
|
||||
"youtu.be",
|
||||
"youtube.com",
|
||||
"zemanta.com",
|
||||
"zoro.to",
|
||||
}
|
||||
67
spn/patrol/domains_test.go
Normal file
67
spn/patrol/domains_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package patrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var enableDomainTools = "no" // change to "yes" to enable
|
||||
|
||||
// TestCleanDomains checks, cleans and prints an improved domain list.
|
||||
// Run with:
|
||||
// go test -run ^TestCleanDomains$ github.com/safing/portmaster/spn/patrol -ldflags "-X github.com/safing/portmaster/spn/patrol.enableDomainTools=yes" -timeout 3h -v
|
||||
// This is provided as a test for easier maintenance and ops.
|
||||
func TestCleanDomains(t *testing.T) { //nolint:paralleltest
|
||||
if enableDomainTools != "yes" {
|
||||
t.Skip()
|
||||
return
|
||||
}
|
||||
|
||||
// Setup context.
|
||||
ctx := context.Background()
|
||||
|
||||
// Go through all domains and check if they are reachable.
|
||||
goodDomains := make([]string, 0, len(testDomains))
|
||||
for _, domain := range testDomains {
|
||||
// Check if domain is reachable.
|
||||
code, err := domainIsUsable(ctx, domain)
|
||||
if err != nil {
|
||||
fmt.Printf("FAIL: %s: %s\n", domain, err)
|
||||
} else {
|
||||
fmt.Printf("OK: %s [%d]\n", domain, code)
|
||||
goodDomains = append(goodDomains, domain)
|
||||
continue
|
||||
}
|
||||
|
||||
// If failed, try again with a www. prefix
|
||||
wwwDomain := "www." + domain
|
||||
code, err = domainIsUsable(ctx, wwwDomain)
|
||||
if err != nil {
|
||||
fmt.Printf("FAIL: %s: %s\n", wwwDomain, err)
|
||||
} else {
|
||||
fmt.Printf("OK: %s [%d]\n", wwwDomain, code)
|
||||
goodDomains = append(goodDomains, wwwDomain)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sort.Strings(goodDomains)
|
||||
fmt.Println("printing good domains:")
|
||||
for _, domain := range goodDomains {
|
||||
fmt.Printf("%q,\n", domain)
|
||||
}
|
||||
|
||||
fmt.Println("IMPORTANT: do not forget to go through list and check if everything looks good")
|
||||
}
|
||||
|
||||
func domainIsUsable(ctx context.Context, domain string) (statusCode int, err error) {
|
||||
// Try IPv6 first as it is way more likely to fail.
|
||||
statusCode, err = CheckHTTPSConnection(ctx, "tcp6", domain)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return CheckHTTPSConnection(ctx, "tcp4", domain)
|
||||
}
|
||||
186
spn/patrol/http.go
Normal file
186
spn/patrol/http.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package patrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/tevino/abool"
|
||||
|
||||
"github.com/safing/portbase/log"
|
||||
"github.com/safing/portbase/modules"
|
||||
"github.com/safing/portmaster/spn/conf"
|
||||
)
|
||||
|
||||
var httpsConnectivityConfirmed = abool.NewBool(true)
|
||||
|
||||
// HTTPSConnectivityConfirmed returns whether the last HTTPS connectivity check succeeded.
|
||||
// Is "true" before first test.
|
||||
func HTTPSConnectivityConfirmed() bool {
|
||||
return httpsConnectivityConfirmed.IsSet()
|
||||
}
|
||||
|
||||
func connectivityCheckTask(ctx context.Context, task *modules.Task) error {
|
||||
// Start tracing logs.
|
||||
ctx, tracer := log.AddTracer(ctx)
|
||||
defer tracer.Submit()
|
||||
|
||||
// Run checks and report status.
|
||||
success := runConnectivityChecks(ctx)
|
||||
if success {
|
||||
tracer.Info("spn/patrol: all connectivity checks succeeded")
|
||||
if httpsConnectivityConfirmed.SetToIf(false, true) {
|
||||
module.TriggerEvent(ChangeSignalEventName, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tracer.Errorf("spn/patrol: connectivity check failed")
|
||||
if httpsConnectivityConfirmed.SetToIf(true, false) {
|
||||
module.TriggerEvent(ChangeSignalEventName, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runConnectivityChecks(ctx context.Context) (ok bool) {
|
||||
switch {
|
||||
case conf.HubHasIPv4() && !runHTTPSConnectivityChecks(ctx, "tcp4"):
|
||||
return false
|
||||
case conf.HubHasIPv6() && !runHTTPSConnectivityChecks(ctx, "tcp6"):
|
||||
return false
|
||||
default:
|
||||
// All checks passed.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func runHTTPSConnectivityChecks(ctx context.Context, network string) (ok bool) {
|
||||
// Step 1: Check 1 domain, require 100%
|
||||
if checkHTTPSConnectivity(ctx, network, 1, 1) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Step 2: Check 5 domains, require 80%
|
||||
if checkHTTPSConnectivity(ctx, network, 5, 0.8) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Step 3: Check 20 domains, require 70%
|
||||
if checkHTTPSConnectivity(ctx, network, 20, 0.7) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func checkHTTPSConnectivity(ctx context.Context, network string, checks int, requiredSuccessFraction float32) (ok bool) {
|
||||
log.Tracer(ctx).Tracef(
|
||||
"spn/patrol: testing connectivity via https (%d checks; %.0f%% required)",
|
||||
checks,
|
||||
requiredSuccessFraction*100,
|
||||
)
|
||||
|
||||
// Run tests.
|
||||
var succeeded int
|
||||
for i := 0; i < checks; i++ {
|
||||
if checkHTTPSConnection(ctx, network) {
|
||||
succeeded++
|
||||
}
|
||||
}
|
||||
|
||||
// Check success.
|
||||
successFraction := float32(succeeded) / float32(checks)
|
||||
if successFraction < requiredSuccessFraction {
|
||||
log.Tracer(ctx).Warningf(
|
||||
"spn/patrol: https/%s connectivity check failed: %d/%d (%.0f%%)",
|
||||
network,
|
||||
succeeded,
|
||||
checks,
|
||||
successFraction*100,
|
||||
)
|
||||
return false
|
||||
}
|
||||
|
||||
log.Tracer(ctx).Debugf(
|
||||
"spn/patrol: https/%s connectivity check succeeded: %d/%d (%.0f%%)",
|
||||
network,
|
||||
succeeded,
|
||||
checks,
|
||||
successFraction*100,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
func checkHTTPSConnection(ctx context.Context, network string) (ok bool) {
|
||||
testDomain := getRandomTestDomain()
|
||||
code, err := CheckHTTPSConnection(ctx, network, testDomain)
|
||||
if err != nil {
|
||||
log.Tracer(ctx).Debugf("spn/patrol: https/%s connect check failed: %s: %s", network, testDomain, err)
|
||||
return false
|
||||
}
|
||||
|
||||
log.Tracer(ctx).Tracef("spn/patrol: https/%s connect check succeeded: %s [%d]", network, testDomain, code)
|
||||
return true
|
||||
}
|
||||
|
||||
// CheckHTTPSConnection checks if a HTTPS connection to the given domain can be established.
|
||||
func CheckHTTPSConnection(ctx context.Context, network, domain string) (statusCode int, err error) {
|
||||
// Check network parameter.
|
||||
switch network {
|
||||
case "tcp4":
|
||||
case "tcp6":
|
||||
default:
|
||||
return 0, fmt.Errorf("provided unsupported network: %s", network)
|
||||
}
|
||||
|
||||
// Build URL.
|
||||
// Use HTTPS to ensure that we have really communicated with the desired
|
||||
// server and not with an intermediate.
|
||||
url := fmt.Sprintf("https://%s/", domain)
|
||||
|
||||
// Prepare all parts of the request.
|
||||
// TODO: Evaluate if we want to change the User-Agent.
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
LocalAddr: conf.GetBindAddr(network),
|
||||
FallbackDelay: -1, // Disables Fast Fallback from IPv6 to IPv4.
|
||||
KeepAlive: -1, // Disable keep-alive.
|
||||
}
|
||||
dialWithNet := func(ctx context.Context, _, addr string) (net.Conn, error) {
|
||||
// Ignore network by http client.
|
||||
// Instead, force either tcp4 or tcp6.
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: dialWithNet,
|
||||
DisableKeepAlives: true,
|
||||
DisableCompression: true,
|
||||
TLSHandshakeTimeout: 15 * time.Second,
|
||||
},
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
// Make request to server.
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to send http request: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||||
return resp.StatusCode, fmt.Errorf("unexpected status code: %s", resp.Status)
|
||||
}
|
||||
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
32
spn/patrol/module.go
Normal file
32
spn/patrol/module.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package patrol
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/safing/portbase/modules"
|
||||
"github.com/safing/portmaster/spn/conf"
|
||||
)
|
||||
|
||||
// ChangeSignalEventName is the name of the event that signals any change in the patrol system.
|
||||
const ChangeSignalEventName = "change signal"
|
||||
|
||||
var module *modules.Module
|
||||
|
||||
func init() {
|
||||
module = modules.Register("patrol", prep, start, nil, "rng")
|
||||
}
|
||||
|
||||
func prep() error {
|
||||
module.RegisterEvent(ChangeSignalEventName, false)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func start() error {
|
||||
if conf.PublicHub() {
|
||||
module.NewTask("connectivity test", connectivityCheckTask).
|
||||
Repeat(5 * time.Minute)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user