github/portmaster
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Restructure modules (#1572)

Browse Source 
* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
2024-08-09 17:15:48 +02:00
committed by GitHub
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
base/rng/doc.go Normal file
View File

@@ -0,0 +1,9 @@
// Package rng provides a feedable CSPRNG.
//
// CSPRNG used is fortuna: github.com/seehuhn/fortuna
// By default the CSPRNG is fed by two sources:
// - It starts with a seed from `crypto/rand` and periodically reseeds from there
// - A really simple tickfeeder which extracts entropy from the internal go scheduler using goroutines and is meant to be used under load.
//
// The RNG can also be easily fed with additional sources.
package rng

124
base/rng/entropy.go Normal file
View File

@@ -0,0 +1,124 @@
package rng
import (
"encoding/binary"
"github.com/tevino/abool"
"github.com/safing/portmaster/service/mgr"
"github.com/safing/structures/container"
)
const (
minFeedEntropy = 256
)
var rngFeeder = make(chan []byte)
// The Feeder is used to feed entropy to the RNG.
type Feeder struct {
input chan *entropyData
entropy int64
needsEntropy *abool.AtomicBool
buffer *container.Container
}
type entropyData struct {
data []byte
entropy int
}
// NewFeeder returns a new entropy Feeder.
func NewFeeder() *Feeder {
newFeeder := &Feeder{
input: make(chan *entropyData),
needsEntropy: abool.NewBool(true),
buffer: container.New(),
}
module.mgr.Go("feeder", newFeeder.run)
return newFeeder
}
// NeedsEntropy returns whether the feeder is currently gathering entropy.
func (f *Feeder) NeedsEntropy() bool {
return f.needsEntropy.IsSet()
}
// SupplyEntropy supplies entropy to the Feeder, it will block until the Feeder has read from it.
func (f *Feeder) SupplyEntropy(data []byte, entropy int) {
f.input <- &entropyData{
data: data,
entropy: entropy,
}
}
// SupplyEntropyIfNeeded supplies entropy to the Feeder, but will not block if no entropy is currently needed.
func (f *Feeder) SupplyEntropyIfNeeded(data []byte, entropy int) {
if f.needsEntropy.IsSet() {
return
}
select {
case f.input <- &entropyData{
data: data,
entropy: entropy,
}:
default:
}
}
// SupplyEntropyAsInt supplies entropy to the Feeder, it will block until the Feeder has read from it.
func (f *Feeder) SupplyEntropyAsInt(n int64, entropy int) {
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, uint64(n))
f.SupplyEntropy(b, entropy)
}
// SupplyEntropyAsIntIfNeeded supplies entropy to the Feeder, but will not block if no entropy is currently needed.
func (f *Feeder) SupplyEntropyAsIntIfNeeded(n int64, entropy int) {
if f.needsEntropy.IsSet() { // avoid allocating a slice if possible
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, uint64(n))
f.SupplyEntropyIfNeeded(b, entropy)
}
}
// CloseFeeder stops the feed processing - the responsible goroutine exits. The input channel is closed and the feeder may not be used anymore in any way.
func (f *Feeder) CloseFeeder() {
close(f.input)
}
func (f *Feeder) run(ctx *mgr.WorkerCtx) error {
defer f.needsEntropy.UnSet()
for {
// gather
f.needsEntropy.Set()
gather:
for {
select {
case newEntropy := <-f.input:
// check if feed has been closed
if newEntropy == nil {
return nil
}
// append to buffer
f.buffer.Append(newEntropy.data)
f.entropy += int64(newEntropy.entropy)
if f.entropy >= minFeedEntropy {
break gather
}
case <-ctx.Done():
return nil
}
}
// feed
f.needsEntropy.UnSet()
select {
case rngFeeder <- f.buffer.CompileData():
case <-ctx.Done():
return nil
}
f.buffer = container.New()
}
}

73
base/rng/entropy_test.go Normal file
View File

@@ -0,0 +1,73 @@
package rng
import (
"testing"
"time"
)
func TestFeeder(t *testing.T) {
t.Parallel()
// wait for start / first round to complete
time.Sleep(1 * time.Millisecond)
f := NewFeeder()
// go through all functions
f.NeedsEntropy()
f.SupplyEntropy([]byte{0}, 0)
f.SupplyEntropyAsInt(0, 0)
f.SupplyEntropyIfNeeded([]byte{0}, 0)
f.SupplyEntropyAsIntIfNeeded(0, 0)
// fill entropy
f.SupplyEntropyAsInt(0, 65535)
// check blocking calls
waitOne := make(chan struct{})
go func() {
f.SupplyEntropy([]byte{0}, 0)
close(waitOne)
}()
select {
case <-waitOne:
t.Error("call does not block!")
case <-time.After(10 * time.Millisecond):
}
waitTwo := make(chan struct{})
go func() {
f.SupplyEntropyAsInt(0, 0)
close(waitTwo)
}()
select {
case <-waitTwo:
t.Error("call does not block!")
case <-time.After(10 * time.Millisecond):
}
// check non-blocking calls
waitThree := make(chan struct{})
go func() {
f.SupplyEntropyIfNeeded([]byte{0}, 0)
close(waitThree)
}()
select {
case <-waitThree:
case <-time.After(10 * time.Millisecond):
t.Error("call blocks!")
}
waitFour := make(chan struct{})
go func() {
f.SupplyEntropyAsIntIfNeeded(0, 0)
close(waitFour)
}()
select {
case <-waitFour:
case <-time.After(10 * time.Millisecond):
t.Error("call blocks!")
}
}

44
base/rng/fullfeed.go Normal file
View File

@@ -0,0 +1,44 @@
package rng
import (
"time"
"github.com/safing/portmaster/service/mgr"
)
func getFullFeedDuration() time.Duration {
// full feed every 5x time of reseedAfterSeconds
secsUntilFullFeed := reseedAfterSeconds * 5
// full feed at most once every ten minutes
if secsUntilFullFeed < 600 {
secsUntilFullFeed = 600
}
return time.Duration(secsUntilFullFeed) * time.Second
}
func fullFeeder(ctx *mgr.WorkerCtx) error {
fullFeedDuration := getFullFeedDuration()
for {
select {
case <-time.After(fullFeedDuration):
rngLock.Lock()
feedAll:
for {
select {
case data := <-rngFeeder:
rng.Reseed(data)
default:
break feedAll
}
}
rngLock.Unlock()
case <-ctx.Done():
return nil
}
}
}

15
base/rng/fullfeed_test.go Normal file
View File

@@ -0,0 +1,15 @@
package rng
import (
"testing"
)
func TestFullFeeder(t *testing.T) {
t.Parallel()
for range 10 {
go func() {
rngFeeder <- []byte{0}
}()
}
}

94
base/rng/get.go Normal file
View File

@@ -0,0 +1,94 @@
package rng
import (
"encoding/binary"
"errors"
"io"
"math"
"time"
)
const (
reseedAfterSeconds = 600 // ten minutes
reseedAfterBytes = 1048576 // one megabyte
)
var (
// Reader provides a global instance to read from the RNG.
Reader io.Reader
rngBytesRead uint64
rngLastFeed = time.Now()
)
// reader provides an io.Reader interface.
type reader struct{}
func init() {
Reader = reader{}
}
func checkEntropy() (err error) {
if !rngReady {
return errors.New("RNG is not ready yet")
}
if rngBytesRead > reseedAfterBytes ||
int(time.Since(rngLastFeed).Seconds()) > reseedAfterSeconds {
select {
case r := <-rngFeeder:
rng.Reseed(r)
rngBytesRead = 0
rngLastFeed = time.Now()
case <-time.After(1 * time.Second):
return errors.New("failed to get new entropy")
}
}
return nil
}
// Read reads random bytes into the supplied byte slice.
func Read(b []byte) (n int, err error) {
rngLock.Lock()
defer rngLock.Unlock()
if err := checkEntropy(); err != nil {
return 0, err
}
return copy(b, rng.PseudoRandomData(uint(len(b)))), nil
}
// Read implements the io.Reader interface.
func (r reader) Read(b []byte) (n int, err error) {
return Read(b)
}
// Bytes allocates a new byte slice of given length and fills it with random data.
func Bytes(n int) ([]byte, error) {
rngLock.Lock()
defer rngLock.Unlock()
if err := checkEntropy(); err != nil {
return nil, err
}
return rng.PseudoRandomData(uint(n)), nil
}
// Number returns a random number from 0 to (incl.) max.
func Number(max uint64) (uint64, error) {
secureLimit := math.MaxUint64 - (math.MaxUint64 % max)
max++
for {
randomBytes, err := Bytes(8)
if err != nil {
return 0, err
}
candidate := binary.LittleEndian.Uint64(randomBytes)
if candidate < secureLimit {
return candidate % max, nil
}
}
}

41
base/rng/get_test.go Normal file
View File

@@ -0,0 +1,41 @@
package rng
import (
"testing"
)
func TestNumberRandomness(t *testing.T) {
t.Parallel()
// skip in automated tests
t.Logf("Integer number bias test deactivated, as it sometimes triggers.")
t.SkipNow()
if testing.Short() {
t.Skip()
}
var subjects uint64 = 10
var testSize uint64 = 10000
results := make([]uint64, int(subjects))
for range int(subjects * testSize) {
n, err := Number(subjects - 1)
if err != nil {
t.Fatal(err)
return
}
results[int(n)]++
}
// catch big mistakes in the number function, eg. massive % bias
lowerMargin := testSize - testSize/50
upperMargin := testSize + testSize/50
for subject, result := range results {
if result < lowerMargin || result > upperMargin {
t.Errorf("subject %d is outside of margins: %d", subject, result)
}
}
t.Fatal(results)
}

36
base/rng/osfeeder.go Normal file
View File

@@ -0,0 +1,36 @@
package rng
import (
"crypto/rand"
"fmt"
"github.com/safing/portmaster/service/mgr"
)
func osFeeder(ctx *mgr.WorkerCtx) error {
entropyBytes := minFeedEntropy / 8
feeder := NewFeeder()
defer feeder.CloseFeeder()
for {
// gather
osEntropy := make([]byte, entropyBytes)
n, err := rand.Read(osEntropy)
if err != nil {
return fmt.Errorf("could not read entropy from os: %w", err)
}
if n != entropyBytes {
return fmt.Errorf("could not read enough entropy from os: got only %d bytes instead of %d", n, entropyBytes)
}
// feed
select {
case feeder.input <- &entropyData{
data: osEntropy,
entropy: entropyBytes * 8,
}:
case <-ctx.Done():
return nil
}
}
}

114
base/rng/rng.go Normal file
View File

@@ -0,0 +1,114 @@
package rng
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/aead/serpent"
"github.com/seehuhn/fortuna"
"github.com/safing/portmaster/service/mgr"
)
// Rng is a random number generator.
type Rng struct {
mgr *mgr.Manager
instance instance
}
var (
rng *fortuna.Generator
rngLock sync.Mutex
rngReady = false
rngCipher = "aes"
// Possible values: "aes", "serpent".
)
func newCipher(key []byte) (cipher.Block, error) {
switch rngCipher {
case "aes":
return aes.NewCipher(key)
case "serpent":
return serpent.NewCipher(key)
default:
return nil, fmt.Errorf("unknown or unsupported cipher: %s", rngCipher)
}
}
// Manager returns the module manager.
func (r *Rng) Manager() *mgr.Manager {
return r.mgr
}
// Start starts the module.
func (r *Rng) Start() error {
rngLock.Lock()
defer rngLock.Unlock()
rng = fortuna.NewGenerator(newCipher)
if rng == nil {
return errors.New("failed to initialize rng")
}
// add another (async) OS rng seed
r.mgr.Go("initial rng feed", func(_ *mgr.WorkerCtx) error {
// get entropy from OS
osEntropy := make([]byte, minFeedEntropy/8)
_, err := rand.Read(osEntropy)
if err != nil {
return fmt.Errorf("could not read entropy from os: %w", err)
}
// feed
rngLock.Lock()
rng.Reseed(osEntropy)
rngLock.Unlock()
return nil
})
// mark as ready
rngReady = true
// random source: OS
r.mgr.Go("os rng feeder", osFeeder)
// random source: goroutine ticks
r.mgr.Go("tick rng feeder", tickFeeder)
// full feeder
r.mgr.Go("full feeder", fullFeeder)
return nil
}
// Stop stops the module.
func (r *Rng) Stop() error {
return nil
}
var (
module *Rng
shimLoaded atomic.Bool
)
// New returns a new rng.
func New(instance instance) (*Rng, error) {
if !shimLoaded.CompareAndSwap(false, true) {
return nil, errors.New("only one instance allowed")
}
m := mgr.New("Rng")
module = &Rng{
mgr: m,
instance: instance,
}
return module, nil
}
type instance interface{}

56
base/rng/rng_test.go Normal file
View File

@@ -0,0 +1,56 @@
package rng
import (
"testing"
)
func init() {
var err error
module, err = New(struct{}{})
if err != nil {
panic(err)
}
err = module.Start()
if err != nil {
panic(err)
}
}
func TestRNG(t *testing.T) {
t.Parallel()
key := make([]byte, 16)
rngCipher = "aes"
_, err := newCipher(key)
if err != nil {
t.Errorf("failed to create aes cipher: %s", err)
}
rngCipher = "serpent"
_, err = newCipher(key)
if err != nil {
t.Errorf("failed to create serpent cipher: %s", err)
}
b := make([]byte, 32)
_, err = Read(b)
if err != nil {
t.Errorf("Read failed: %s", err)
}
_, err = Reader.Read(b)
if err != nil {
t.Errorf("Read failed: %s", err)
}
_, err = Bytes(32)
if err != nil {
t.Errorf("Bytes failed: %s", err)
}
_, err = Number(100)
if err != nil {
t.Errorf("Number failed: %s", err)
}
}

4
base/rng/test/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
test
*.bin
*.out
*.txt

279
base/rng/test/README.md Normal file
View File

@@ -0,0 +1,279 @@
# Entropy Testing
In order verify that the random package actually generates random enough entropy/data, this test program holds the core functions that generate entropy as well as some noise makers to simulate a running program.
Please also note that output from `tickFeeder` is never used directly but fed as entropy to the actual RNG - `fortuna`.
With `tickFeeder`, to be sure that the delivered entropy is of high enough quality, only 1 bit of entropy is expected per generated byte - ie. we gather 8 times the amount we need. The following test below is run on the raw output.
To test the quality of entropy, first generate random data with the test program:
go build
./test tickfeeder tickfeeder.out 1 # just the additional entropy feed
# OR
./test fortuna fortuna.out 10 # the actual CSPRNG with feeders
Then, run `dieharder`, a random number generator test tool:
dieharder -a -f output.bin
Below you can find two test outputs of `dieharder`.
Please note that around 5 tests of `dieharder` normally fail. This is expected and even desired.
Also, the rng currently reseeds (ie. adds entropy) after 1MB or 10 minutes.
`dieharder` of two samples of 10MB of fortuna (with feeders) (`go version go1.14.2 linux/amd64` on 21.04.2020):
#=============================================================================#
# dieharder version 3.31.1 Copyright 2003 Robert G. Brown #
#=============================================================================#
rng_name | filename |rands/second|
mt19937| fortuna.out| 1.00e+08 |
#=============================================================================#
test_name |ntup| tsamples |psamples| p-value |Assessment
#=============================================================================#
diehard_birthdays| 0| 100| 100|0.69048981| PASSED | 2nd sample: PASSED
diehard_operm5| 0| 1000000| 100|0.76010702| PASSED | 2nd sample: PASSED
diehard_rank_32x32| 0| 40000| 100|0.86291558| PASSED | 2nd sample: PASSED
diehard_rank_6x8| 0| 100000| 100|0.63715647| PASSED | 2nd sample: PASSED
diehard_bitstream| 0| 2097152| 100|0.25389670| PASSED | 2nd sample: PASSED
diehard_opso| 0| 2097152| 100|0.70928590| PASSED | 2nd sample: PASSED
diehard_oqso| 0| 2097152| 100|0.75643141| PASSED | 2nd sample: PASSED
diehard_dna| 0| 2097152| 100|0.57096286| PASSED | 2nd sample: PASSED
diehard_count_1s_str| 0| 256000| 100|0.39650366| PASSED | 2nd sample: PASSED
diehard_count_1s_byt| 0| 256000| 100|0.26040557| PASSED | 2nd sample: PASSED
diehard_parking_lot| 0| 12000| 100|0.92327672| PASSED | 2nd sample: PASSED
diehard_2dsphere| 2| 8000| 100|0.86507605| PASSED | 2nd sample: PASSED
diehard_3dsphere| 3| 4000| 100|0.70845388| PASSED | 2nd sample: PASSED
diehard_squeeze| 0| 100000| 100|0.99744782| WEAK | 2nd sample: PASSED
diehard_sums| 0| 100| 100|0.27275938| PASSED | 2nd sample: PASSED
diehard_runs| 0| 100000| 100|0.27299936| PASSED | 2nd sample: PASSED
diehard_runs| 0| 100000| 100|0.42043270| PASSED | 2nd sample: PASSED
diehard_craps| 0| 200000| 100|0.91674884| PASSED | 2nd sample: PASSED
diehard_craps| 0| 200000| 100|0.77856237| PASSED | 2nd sample: PASSED
marsaglia_tsang_gcd| 0| 10000000| 100|0.77922797| PASSED | 2nd sample: PASSED
marsaglia_tsang_gcd| 0| 10000000| 100|0.94589532| PASSED | 2nd sample: PASSED
sts_monobit| 1| 100000| 100|0.99484549| PASSED | 2nd sample: PASSED
sts_runs| 2| 100000| 100|0.70036713| PASSED | 2nd sample: PASSED
sts_serial| 1| 100000| 100|0.79544015| PASSED | 2nd sample: PASSED
sts_serial| 2| 100000| 100|0.91473958| PASSED | 2nd sample: PASSED
sts_serial| 3| 100000| 100|0.66528037| PASSED | 2nd sample: PASSED
sts_serial| 3| 100000| 100|0.84028312| PASSED | 2nd sample: PASSED
sts_serial| 4| 100000| 100|0.82253130| PASSED | 2nd sample: PASSED
sts_serial| 4| 100000| 100|0.90695315| PASSED | 2nd sample: PASSED
sts_serial| 5| 100000| 100|0.55160515| PASSED | 2nd sample: PASSED
sts_serial| 5| 100000| 100|0.05256789| PASSED | 2nd sample: PASSED
sts_serial| 6| 100000| 100|0.25857850| PASSED | 2nd sample: PASSED
sts_serial| 6| 100000| 100|0.58661649| PASSED | 2nd sample: PASSED
sts_serial| 7| 100000| 100|0.46915559| PASSED | 2nd sample: PASSED
sts_serial| 7| 100000| 100|0.57273130| PASSED | 2nd sample: PASSED
sts_serial| 8| 100000| 100|0.99182961| PASSED | 2nd sample: PASSED
sts_serial| 8| 100000| 100|0.86913367| PASSED | 2nd sample: PASSED
sts_serial| 9| 100000| 100|0.19259756| PASSED | 2nd sample: PASSED
sts_serial| 9| 100000| 100|0.61225842| PASSED | 2nd sample: PASSED
sts_serial| 10| 100000| 100|0.40792308| PASSED | 2nd sample: PASSED
sts_serial| 10| 100000| 100|0.99930785| WEAK | 2nd sample: PASSED
sts_serial| 11| 100000| 100|0.07296973| PASSED | 2nd sample: PASSED
sts_serial| 11| 100000| 100|0.04906522| PASSED | 2nd sample: PASSED
sts_serial| 12| 100000| 100|0.66400927| PASSED | 2nd sample: PASSED
sts_serial| 12| 100000| 100|0.67947609| PASSED | 2nd sample: PASSED
sts_serial| 13| 100000| 100|0.20412325| PASSED | 2nd sample: PASSED
sts_serial| 13| 100000| 100|0.19781734| PASSED | 2nd sample: PASSED
sts_serial| 14| 100000| 100|0.08541533| PASSED | 2nd sample: PASSED
sts_serial| 14| 100000| 100|0.07438464| PASSED | 2nd sample: PASSED
sts_serial| 15| 100000| 100|0.04607276| PASSED | 2nd sample: PASSED
sts_serial| 15| 100000| 100|0.56460340| PASSED | 2nd sample: PASSED
sts_serial| 16| 100000| 100|0.40211405| PASSED | 2nd sample: PASSED
sts_serial| 16| 100000| 100|0.81369172| PASSED | 2nd sample: PASSED
rgb_bitdist| 1| 100000| 100|0.52317549| PASSED | 2nd sample: PASSED
rgb_bitdist| 2| 100000| 100|0.49819655| PASSED | 2nd sample: PASSED
rgb_bitdist| 3| 100000| 100|0.65830167| PASSED | 2nd sample: PASSED
rgb_bitdist| 4| 100000| 100|0.75278398| PASSED | 2nd sample: PASSED
rgb_bitdist| 5| 100000| 100|0.23537303| PASSED | 2nd sample: PASSED
rgb_bitdist| 6| 100000| 100|0.82461608| PASSED | 2nd sample: PASSED
rgb_bitdist| 7| 100000| 100|0.46944789| PASSED | 2nd sample: PASSED
rgb_bitdist| 8| 100000| 100|0.44371293| PASSED | 2nd sample: PASSED
rgb_bitdist| 9| 100000| 100|0.61647469| PASSED | 2nd sample: PASSED
rgb_bitdist| 10| 100000| 100|0.97623808| PASSED | 2nd sample: PASSED
rgb_bitdist| 11| 100000| 100|0.26037998| PASSED | 2nd sample: PASSED
rgb_bitdist| 12| 100000| 100|0.59217788| PASSED | 2nd sample: PASSED
rgb_minimum_distance| 2| 10000| 1000|0.19809129| PASSED | 2nd sample: PASSED
rgb_minimum_distance| 3| 10000| 1000|0.97363365| PASSED | 2nd sample: PASSED
rgb_minimum_distance| 4| 10000| 1000|0.62281709| PASSED | 2nd sample: PASSED
rgb_minimum_distance| 5| 10000| 1000|0.13655852| PASSED | 2nd sample: PASSED
rgb_permutations| 2| 100000| 100|0.33726465| PASSED | 2nd sample: PASSED
rgb_permutations| 3| 100000| 100|0.21992025| PASSED | 2nd sample: WEAK
rgb_permutations| 4| 100000| 100|0.27074573| PASSED | 2nd sample: PASSED
rgb_permutations| 5| 100000| 100|0.76925248| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 0| 1000000| 100|0.91881971| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 1| 1000000| 100|0.08282106| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 2| 1000000| 100|0.55991289| PASSED | 2nd sample: WEAK
rgb_lagged_sum| 3| 1000000| 100|0.94939920| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 4| 1000000| 100|0.21248759| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 5| 1000000| 100|0.99308883| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 6| 1000000| 100|0.83174944| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 7| 1000000| 100|0.49883983| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 8| 1000000| 100|0.99900807| WEAK | 2nd sample: PASSED
rgb_lagged_sum| 9| 1000000| 100|0.74164128| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 10| 1000000| 100|0.53367081| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 11| 1000000| 100|0.41808417| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 12| 1000000| 100|0.96082733| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 13| 1000000| 100|0.38208924| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 14| 1000000| 100|0.98335747| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 15| 1000000| 100|0.68708033| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 16| 1000000| 100|0.49715110| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 17| 1000000| 100|0.68418225| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 18| 1000000| 100|0.97255087| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 19| 1000000| 100|0.99556843| WEAK | 2nd sample: PASSED
rgb_lagged_sum| 20| 1000000| 100|0.50758123| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 21| 1000000| 100|0.98435826| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 22| 1000000| 100|0.15752743| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 23| 1000000| 100|0.98607886| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 24| 1000000| 100|0.86645723| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 25| 1000000| 100|0.87384758| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 26| 1000000| 100|0.98680940| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 27| 1000000| 100|0.56386729| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 28| 1000000| 100|0.16874165| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 29| 1000000| 100|0.10369211| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 30| 1000000| 100|0.91356341| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 31| 1000000| 100|0.42526940| PASSED | 2nd sample: PASSED
rgb_lagged_sum| 32| 1000000| 100|0.99939460| WEAK | 2nd sample: PASSED
rgb_kstest_test| 0| 10000| 1000|0.11414525| PASSED | 2nd sample: PASSED
dab_bytedistrib| 0| 51200000| 1|0.27693890| PASSED | 2nd sample: PASSED
dab_dct| 256| 50000| 1|0.15807123| PASSED | 2nd sample: PASSED
Preparing to run test 207. ntuple = 0
dab_filltree| 32| 15000000| 1|0.33275771| PASSED | 2nd sample: PASSED
dab_filltree| 32| 15000000| 1|0.15704033| PASSED | 2nd sample: PASSED
Preparing to run test 208. ntuple = 0
dab_filltree2| 0| 5000000| 1|0.85562670| PASSED | 2nd sample: PASSED
dab_filltree2| 1| 5000000| 1|0.35187836| PASSED | 2nd sample: PASSED
Preparing to run test 209. ntuple = 0
dab_monobit2| 12| 65000000| 1|0.03099468| PASSED | 2nd sample: PASSED
`dieharder` output of 22KB of contextswitch (`go version go1.10.3 linux/amd64` on 23.08.2018):
#=============================================================================#
# dieharder version 3.31.1 Copyright 2003 Robert G. Brown #
#=============================================================================#
rng_name | filename |rands/second|
mt19937| output.bin| 1.00e+08 |
#=============================================================================#
test_name |ntup| tsamples |psamples| p-value |Assessment
#=============================================================================#
diehard_birthdays| 0| 100| 100|0.75124818| PASSED
diehard_operm5| 0| 1000000| 100|0.71642114| PASSED
diehard_rank_32x32| 0| 40000| 100|0.66406749| PASSED
diehard_rank_6x8| 0| 100000| 100|0.79742497| PASSED
diehard_bitstream| 0| 2097152| 100|0.68336079| PASSED
diehard_opso| 0| 2097152| 100|0.99670345| WEAK
diehard_oqso| 0| 2097152| 100|0.85930861| PASSED
diehard_dna| 0| 2097152| 100|0.77857540| PASSED
diehard_count_1s_str| 0| 256000| 100|0.27851730| PASSED
diehard_count_1s_byt| 0| 256000| 100|0.29570009| PASSED
diehard_parking_lot| 0| 12000| 100|0.51526020| PASSED
diehard_2dsphere| 2| 8000| 100|0.49199324| PASSED
diehard_3dsphere| 3| 4000| 100|0.99008122| PASSED
diehard_squeeze| 0| 100000| 100|0.95518110| PASSED
diehard_sums| 0| 100| 100|0.00015930| WEAK
diehard_runs| 0| 100000| 100|0.50091086| PASSED
diehard_runs| 0| 100000| 100|0.44091340| PASSED
diehard_craps| 0| 200000| 100|0.77284264| PASSED
diehard_craps| 0| 200000| 100|0.71027434| PASSED
marsaglia_tsang_gcd| 0| 10000000| 100|0.38138922| PASSED
marsaglia_tsang_gcd| 0| 10000000| 100|0.36661590| PASSED
sts_monobit| 1| 100000| 100|0.06209802| PASSED
sts_runs| 2| 100000| 100|0.82506539| PASSED
sts_serial| 1| 100000| 100|0.99198615| PASSED
sts_serial| 2| 100000| 100|0.85604831| PASSED
sts_serial| 3| 100000| 100|0.06613657| PASSED
sts_serial| 3| 100000| 100|0.16787860| PASSED
sts_serial| 4| 100000| 100|0.45227401| PASSED
sts_serial| 4| 100000| 100|0.43529092| PASSED
sts_serial| 5| 100000| 100|0.99912474| WEAK
sts_serial| 5| 100000| 100|0.94754128| PASSED
sts_serial| 6| 100000| 100|0.98406523| PASSED
sts_serial| 6| 100000| 100|0.92895983| PASSED
sts_serial| 7| 100000| 100|0.45965410| PASSED
sts_serial| 7| 100000| 100|0.64185152| PASSED
sts_serial| 8| 100000| 100|0.57922926| PASSED
sts_serial| 8| 100000| 100|0.52390292| PASSED
sts_serial| 9| 100000| 100|0.82722325| PASSED
sts_serial| 9| 100000| 100|0.89384819| PASSED
sts_serial| 10| 100000| 100|0.79877889| PASSED
sts_serial| 10| 100000| 100|0.49562348| PASSED
sts_serial| 11| 100000| 100|0.09217966| PASSED
sts_serial| 11| 100000| 100|0.00342361| WEAK
sts_serial| 12| 100000| 100|0.60119444| PASSED
sts_serial| 12| 100000| 100|0.20420318| PASSED
sts_serial| 13| 100000| 100|0.76867489| PASSED
sts_serial| 13| 100000| 100|0.35717970| PASSED
sts_serial| 14| 100000| 100|0.67364089| PASSED
sts_serial| 14| 100000| 100|0.98667204| PASSED
sts_serial| 15| 100000| 100|0.24328833| PASSED
sts_serial| 15| 100000| 100|0.52098866| PASSED
sts_serial| 16| 100000| 100|0.48845863| PASSED
sts_serial| 16| 100000| 100|0.61943558| PASSED
rgb_bitdist| 1| 100000| 100|0.24694812| PASSED
rgb_bitdist| 2| 100000| 100|0.75873723| PASSED
rgb_bitdist| 3| 100000| 100|0.28670990| PASSED
rgb_bitdist| 4| 100000| 100|0.41966273| PASSED
rgb_bitdist| 5| 100000| 100|0.80463973| PASSED
rgb_bitdist| 6| 100000| 100|0.44747725| PASSED
rgb_bitdist| 7| 100000| 100|0.35848420| PASSED
rgb_bitdist| 8| 100000| 100|0.56585089| PASSED
rgb_bitdist| 9| 100000| 100|0.23179559| PASSED
rgb_bitdist| 10| 100000| 100|0.83369283| PASSED
rgb_bitdist| 11| 100000| 100|0.74761235| PASSED
rgb_bitdist| 12| 100000| 100|0.50477673| PASSED
rgb_minimum_distance| 2| 10000| 1000|0.29527530| PASSED
rgb_minimum_distance| 3| 10000| 1000|0.83681186| PASSED
rgb_minimum_distance| 4| 10000| 1000|0.85939646| PASSED
rgb_minimum_distance| 5| 10000| 1000|0.90229335| PASSED
rgb_permutations| 2| 100000| 100|0.99010460| PASSED
rgb_permutations| 3| 100000| 100|0.99360922| PASSED
rgb_permutations| 4| 100000| 100|0.30113906| PASSED
rgb_permutations| 5| 100000| 100|0.60701235| PASSED
rgb_lagged_sum| 0| 1000000| 100|0.37080580| PASSED
rgb_lagged_sum| 1| 1000000| 100|0.91852932| PASSED
rgb_lagged_sum| 2| 1000000| 100|0.74568323| PASSED
rgb_lagged_sum| 3| 1000000| 100|0.64070201| PASSED
rgb_lagged_sum| 4| 1000000| 100|0.53802729| PASSED
rgb_lagged_sum| 5| 1000000| 100|0.67865656| PASSED
rgb_lagged_sum| 6| 1000000| 100|0.85161494| PASSED
rgb_lagged_sum| 7| 1000000| 100|0.37312323| PASSED
rgb_lagged_sum| 8| 1000000| 100|0.17841759| PASSED
rgb_lagged_sum| 9| 1000000| 100|0.85795513| PASSED
rgb_lagged_sum| 10| 1000000| 100|0.79843176| PASSED
rgb_lagged_sum| 11| 1000000| 100|0.21320830| PASSED
rgb_lagged_sum| 12| 1000000| 100|0.94709672| PASSED
rgb_lagged_sum| 13| 1000000| 100|0.12600611| PASSED
rgb_lagged_sum| 14| 1000000| 100|0.26780352| PASSED
rgb_lagged_sum| 15| 1000000| 100|0.07862730| PASSED
rgb_lagged_sum| 16| 1000000| 100|0.21102254| PASSED
rgb_lagged_sum| 17| 1000000| 100|0.82967141| PASSED
rgb_lagged_sum| 18| 1000000| 100|0.05818566| PASSED
rgb_lagged_sum| 19| 1000000| 100|0.01010140| PASSED
rgb_lagged_sum| 20| 1000000| 100|0.17941782| PASSED
rgb_lagged_sum| 21| 1000000| 100|0.98442639| PASSED
rgb_lagged_sum| 22| 1000000| 100|0.30352772| PASSED
rgb_lagged_sum| 23| 1000000| 100|0.56855155| PASSED
rgb_lagged_sum| 24| 1000000| 100|0.27280405| PASSED
rgb_lagged_sum| 25| 1000000| 100|0.41141889| PASSED
rgb_lagged_sum| 26| 1000000| 100|0.25389013| PASSED
rgb_lagged_sum| 27| 1000000| 100|0.10313177| PASSED
rgb_lagged_sum| 28| 1000000| 100|0.76610028| PASSED
rgb_lagged_sum| 29| 1000000| 100|0.97903830| PASSED
rgb_lagged_sum| 30| 1000000| 100|0.51216732| PASSED
rgb_lagged_sum| 31| 1000000| 100|0.98578832| PASSED
rgb_lagged_sum| 32| 1000000| 100|0.95078719| PASSED
rgb_kstest_test| 0| 10000| 1000|0.24930712| PASSED
dab_bytedistrib| 0| 51200000| 1|0.51100031| PASSED
dab_dct| 256| 50000| 1|0.28794956| PASSED
Preparing to run test 207. ntuple = 0
dab_filltree| 32| 15000000| 1|0.93283449| PASSED
dab_filltree| 32| 15000000| 1|0.36488075| PASSED
Preparing to run test 208. ntuple = 0
dab_filltree2| 0| 5000000| 1|0.94036105| PASSED
dab_filltree2| 1| 5000000| 1|0.30118240| PASSED
Preparing to run test 209. ntuple = 0
dab_monobit2| 12| 65000000| 1|0.00209003| WEAK

230
base/rng/test/main.go Normal file
View File

@@ -0,0 +1,230 @@
package main
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"runtime"
"strconv"
"sync/atomic"
"time"
"github.com/safing/portmaster/base/log"
"github.com/safing/portmaster/base/rng"
"github.com/safing/portmaster/service/mgr"
)
// Test tests the rng.
type Test struct {
mgr *mgr.Manager
instance instance
}
var (
module *Test
shimLoaded atomic.Bool
outputFile *os.File
outputSize uint64 = 1000000
)
func init() {
// module = modules.Register("main", prep, start, nil, "rng")
}
func main() {
runtime.GOMAXPROCS(1)
var err error
module, err = New(struct{}{})
if err != nil {
fmt.Printf("failed to initialize module: %s", err)
return
}
err = start()
if err != nil {
fmt.Printf("failed to initialize module: %s", err)
return
}
}
func prep() error {
if len(os.Args) < 3 {
return fmt.Errorf("usage: ./%s {fortuna|tickfeeder} <file> [output size in MB]", os.Args[0])
}
switch os.Args[1] {
case "fortuna":
case "tickfeeder":
default:
return fmt.Errorf("usage: %s {fortuna|tickfeeder}", os.Args[0])
}
if len(os.Args) > 3 {
n, err := strconv.ParseUint(os.Args[3], 10, 64)
if err != nil {
return fmt.Errorf("failed to parse output size: %w", err)
}
outputSize = n * 1000000
}
var err error
outputFile, err = os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0o0644) //nolint:gosec
if err != nil {
return fmt.Errorf("failed to open output file: %w", err)
}
return nil
}
//nolint:gocognit
func start() error {
// generates 1MB and writes to stdout
log.Infof("writing %dMB to stdout, a \".\" will be printed at every 1024 bytes.", outputSize/1000000)
switch os.Args[1] {
case "fortuna":
module.mgr.Go("fortuna", fortuna)
case "tickfeeder":
module.mgr.Go("noise", noise)
module.mgr.Go("tickfeeder", tickfeeder)
default:
return fmt.Errorf("usage: ./%s {fortuna|tickfeeder}", os.Args[0])
}
return nil
}
func fortuna(_ *mgr.WorkerCtx) error {
var bytesWritten uint64
for {
if module.mgr.IsDone() {
return nil
}
b, err := rng.Bytes(64)
if err != nil {
return err
}
_, err = outputFile.Write(b)
if err != nil {
return err
}
bytesWritten += 64
if bytesWritten%1024 == 0 {
_, _ = os.Stderr.WriteString(".")
}
if bytesWritten%65536 == 0 {
fmt.Fprintf(os.Stderr, "\n%d bytes written\n", bytesWritten)
}
if bytesWritten >= outputSize {
_, _ = os.Stderr.WriteString("\n")
break
}
}
go module.mgr.Cancel() //nolint:errcheck
return nil
}
func tickfeeder(ctx *mgr.WorkerCtx) error {
var bytesWritten uint64
var value int64
var pushes int
for {
if module.mgr.IsDone() {
return nil
}
time.Sleep(10 * time.Nanosecond)
value = (value << 1) | (time.Now().UnixNano() % 2)
pushes++
if pushes >= 64 {
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, uint64(value))
_, err := outputFile.Write(b)
if err != nil {
return err
}
bytesWritten += 8
if bytesWritten%1024 == 0 {
_, _ = os.Stderr.WriteString(".")
}
if bytesWritten%65536 == 0 {
fmt.Fprintf(os.Stderr, "\n%d bytes written\n", bytesWritten)
}
pushes = 0
}
if bytesWritten >= outputSize {
_, _ = os.Stderr.WriteString("\n")
break
}
}
go module.mgr.Cancel() //nolint:errcheck
return nil
}
func noise(ctx *mgr.WorkerCtx) error {
// do some aes ctr for noise
key, _ := hex.DecodeString("6368616e676520746869732070617373")
data := []byte("some plaintext x")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
iv := make([]byte, aes.BlockSize)
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
stream := cipher.NewCTR(block, iv)
for {
select {
case <-ctx.Done():
return nil
default:
stream.XORKeyStream(data, data)
}
}
}
// New returns a new rng test.
func New(instance instance) (*Test, error) {
if !shimLoaded.CompareAndSwap(false, true) {
return nil, errors.New("only one instance allowed")
}
m := mgr.New("geoip")
module = &Test{
mgr: m,
instance: instance,
}
if err := prep(); err != nil {
return nil, err
}
return module, nil
}
type instance interface{}

76
base/rng/tickfeeder.go Normal file
View File

@@ -0,0 +1,76 @@
package rng
import (
"encoding/binary"
"time"
"github.com/safing/portmaster/service/mgr"
)
func getTickFeederTickDuration() time.Duration {
// be ready in 1/10 time of reseedAfterSeconds
msecsAvailable := reseedAfterSeconds * 100
// ex.: reseed after 10 minutes: msecsAvailable = 60000
// have full entropy after 5 minutes
// one tick generates 0,125 bits of entropy
ticksNeeded := minFeedEntropy * 8
// ex.: minimum entropy is 256: ticksNeeded = 2048
// msces between ticks
tickMsecs := msecsAvailable / ticksNeeded
// ex.: tickMsecs = 29(,296875)
// use a minimum of 10 msecs per tick for good entropy
// it would take 21 seconds to get full 256 bits of entropy with 10msec ticks
if tickMsecs < 10 {
tickMsecs = 10
}
return time.Duration(tickMsecs) * time.Millisecond
}
// tickFeeder is a really simple entropy feeder that adds the least significant bit of the current nanosecond unixtime to its pool every time it 'ticks'.
// The more work the program does, the better the quality, as the internal schedular cannot immediately run the goroutine when it's ready.
func tickFeeder(ctx *mgr.WorkerCtx) error {
var value int64
var pushes int
feeder := NewFeeder()
defer feeder.CloseFeeder()
tickDuration := getTickFeederTickDuration()
for {
// wait for tick
time.Sleep(tickDuration)
// add tick value
value = (value << 1) | (time.Now().UnixNano() % 2)
pushes++
if pushes >= 64 {
// convert to []byte
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, uint64(value))
// reset
pushes = 0
// feed
select {
case feeder.input <- &entropyData{
data: b,
entropy: 8,
}:
case <-ctx.Done():
return nil
}
} else {
// check if are done
select {
case <-ctx.Done():
return nil
default:
}
}
}
}