diff --git a/go.mod b/go.mod index 997bdcb0..3100d1f9 100644 --- a/go.mod +++ b/go.mod @@ -1,80 +1,34 @@ module github.com/safing/portmaster -go 1.18 +go 1.15 require ( github.com/agext/levenshtein v1.2.3 github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d github.com/coreos/go-iptables v0.6.0 - github.com/florianl/go-nfqueue v1.3.1 + github.com/florianl/go-nfqueue v1.3.0 github.com/godbus/dbus/v5 v5.1.0 github.com/google/gopacket v1.1.19 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.4.0 - github.com/miekg/dns v1.1.49 - github.com/oschwald/maxminddb-golang v1.9.0 - github.com/safing/portbase v0.14.4 - github.com/safing/spn v0.4.11 + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/mdlayher/socket v0.2.2 // indirect + github.com/miekg/dns v1.1.46 + github.com/oschwald/maxminddb-golang v1.8.0 + github.com/safing/portbase v0.14.0 + github.com/safing/spn v0.4.3 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/spf13/cobra v1.4.0 - github.com/stretchr/testify v1.7.1 + github.com/spf13/cobra v1.3.0 + github.com/stretchr/testify v1.7.0 github.com/tannerryan/ring v1.1.2 github.com/tevino/abool v1.2.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - golang.org/x/net v0.0.0-20220513224357-95641704303c - golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 - golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a + golang.org/x/net v0.0.0-20220225172249-27dd8689420f + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 + zombiezen.com/go/sqlite v0.9.2 ) -require ( - github.com/VictoriaMetrics/metrics v1.18.1 // indirect - github.com/aead/ecdh v0.2.0 // indirect - github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 // indirect - github.com/armon/go-radix v1.0.0 // indirect - github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect - github.com/bluele/gcache v0.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/gofrs/uuid v4.2.0+incompatible // indirect - github.com/google/go-cmp v0.5.8 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/josharian/native v1.0.0 // indirect - github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/mdlayher/netlink v1.6.0 // indirect - github.com/mdlayher/socket v0.2.3 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rot256/pblind v0.0.0-20211117203330-22455f90b565 // indirect - github.com/safing/jess v0.2.3 // indirect - github.com/satori/go.uuid v1.2.0 // indirect - github.com/seehuhn/fortuna v1.0.1 // indirect - github.com/seehuhn/sha256d v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.14.1 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect - github.com/tidwall/sjson v1.2.4 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.5.0 // indirect - github.com/valyala/fastrand v1.1.0 // indirect - github.com/valyala/histogram v1.2.0 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/x448/float16 v0.8.4 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/tools v0.1.10 // indirect - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect -) +replace github.com/safing/spn => ../spn + +replace github.com/safing/portbase => ../portbase diff --git a/go.sum b/go.sum index 7fe99705..390453ed 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,7 @@ cloud.google.com/go/trace v0.1.0/go.mod h1:wxEwsoeRVPbeSkt7ZC9nWCgmoKQRAoySN7XHW contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= +crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AlecAivazis/survey/v2 v2.0.7/go.mod h1:mlizQTaPjnR4jcpwRSaSlkbsRfYFEyKgLQvYTzxxiHA= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= @@ -459,6 +460,7 @@ github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -473,6 +475,7 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -646,7 +649,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE= @@ -788,6 +793,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -969,6 +976,7 @@ github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp1 github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= @@ -1082,6 +1090,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1131,6 +1140,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= @@ -1314,6 +1324,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1350,6 +1361,7 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1455,6 +1467,7 @@ golang.org/x/tools v0.0.0-20200808161706-5bf02b21f123/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1466,6 +1479,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= @@ -1696,6 +1710,129 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60= +modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw= +modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI= +modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag= +modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw= +modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ= +modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c= +modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo= +modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg= +modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I= +modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs= +modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8= +modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE= +modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk= +modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w= +modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE= +modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8= +modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc= +modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU= +modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE= +modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk= +modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI= +modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE= +modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg= +modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74= +modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU= +modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU= +modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc= +modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM= +modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ= +modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84= +modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ= +modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY= +modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w= +modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU= +modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko= +modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA= +modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4= +modernc.org/ccgo/v3 v3.14.0/go.mod h1:hBrkiBlUwvr5vV/ZH9YzXIp982jKE8Ek8tR1ytoAL6Q= +modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0= +modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0= +modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8= +modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I= +modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q= +modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg= +modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M= +modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU= +modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE= +modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso= +modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8= +modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8= +modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I= +modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk= +modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY= +modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE= +modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg= +modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM= +modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg= +modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo= +modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8= +modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ= +modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA= +modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM= +modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg= +modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE= +modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM= +modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU= +modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw= +modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M= +modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18= +modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8= +modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw= +modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0= +modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI= +modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE= +modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY= +modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ= +modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c= +modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI= +modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ= +modernc.org/libc v1.13.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk= +modernc.org/libc v1.13.2/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk= +modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk= +modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34= +modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ= +modernc.org/libc v1.14.5 h1:DAHvwGoVRDZs5iJXnX9RJrgXSsorupCWmJ2ac964Owk= +modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14= +modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.14.5 h1:bYrrjwH9Y7QUGk1MbchZDhRfmpGuEAs/D45sVjNbfvs= +modernc.org/sqlite v1.14.5/go.mod h1:YyX5Rx0WbXokitdWl2GJIDy4BrPxBP0PwwhpXOHCDLE= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/tcl v1.10.0/go.mod h1:WzWapmP/7dHVhFoyPpEaNSVTL8xtewhouN/cqSJ5A2s= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.2.21/go.mod h1:uXrObx4pGqXWIMliC5MiKuwAyMrltzwpteOFUP1PWCc= +modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1705,3 +1842,5 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +zombiezen.com/go/sqlite v0.9.2 h1:bgY6e0BksSrrQqM9gK+HYLbMj1zxcRWHijJBlYnRpCw= +zombiezen.com/go/sqlite v0.9.2/go.mod h1:M/gb7zbJfWDAUQAsw/9wf0c3P1cHt6Mv9zrtVmAs13Y= diff --git a/netquery/orm/decoder.go b/netquery/orm/decoder.go new file mode 100644 index 00000000..5e0d1f7b --- /dev/null +++ b/netquery/orm/decoder.go @@ -0,0 +1,404 @@ +package orm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "reflect" + "strings" + "time" + + "zombiezen.com/go/sqlite" +) + +// Commonly used error messages when working with orm. +var ( + errStructExpected = errors.New("encode: can only encode structs to maps") + errStructPointerExpected = errors.New("decode: result must be pointer to a struct type or map[string]interface{}") + errUnexpectedColumnType = errors.New("decode: unexpected column type") +) + +// constants used when transforming data to and from sqlite. +var ( + // sqliteTimeFromat defines the string representation that is + // expected by SQLite DATETIME functions. + // Note that SQLite itself does not include support for a DATETIME + // column type. Instead, dates and times are stored either as INTEGER, + // TEXT or REAL. + // This package provides support for time.Time being stored as TEXT (using a + // preconfigured timezone; UTC by default) or as INTEGER (the user can choose between + // unixepoch and unixnano-epoch where the nano variant is not offically supported by + // SQLITE). + sqliteTimeFormat = "2006-01-02 15:04:05" +) + +type ( + + // Stmt describes the interface that must be implemented in order to + // be decodable to a struct type using DecodeStmt. This interface is implemented + // by *sqlite.Stmt. + Stmt interface { + ColumnCount() int + ColumnName(int) string + ColumnType(int) sqlite.ColumnType + ColumnText(int) string + ColumnBool(int) bool + ColumnFloat(int) float64 + ColumnInt(int) int + ColumnReader(int) *bytes.Reader + } + + // DecodeFunc is called for each non-basic type during decoding. + DecodeFunc func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) + + DecodeConfig struct { + DecodeHooks []DecodeFunc + } +) + +// DecodeStmt decodes the current result row loaded in Stmt into the struct or map type result. +// Decoding hooks configured in cfg are executed before trying to decode basic types and may +// be specified to provide support for special types. +// See DatetimeDecoder() for an example of a DecodeHook that handles graceful time.Time conversion. +// +func DecodeStmt(ctx context.Context, stmt Stmt, result interface{}, cfg DecodeConfig) error { + // make sure we got something to decode into ... + if result == nil { + return fmt.Errorf("%w, got %T", errStructPointerExpected, result) + } + + // fast path for decoding into a map + if mp, ok := result.(*map[string]interface{}); ok { + return decodeIntoMap(ctx, stmt, mp) + } + + // make sure we got a pointer in result + if reflect.TypeOf(result).Kind() != reflect.Ptr { + return fmt.Errorf("%w, got %T", errStructPointerExpected, result) + } + + // make sure it's a poiter to a struct type + t := reflect.ValueOf(result).Elem().Type() + if t.Kind() != reflect.Struct { + return fmt.Errorf("%w, got %T", errStructPointerExpected, result) + } + + // if result is a nil pointer make sure to allocate some space + // for the resulting struct + resultValue := reflect.ValueOf(result) + if resultValue.IsNil() { + resultValue.Set( + reflect.New(t), + ) + } + + // we need access to the struct directly and not to the + // pointer. + target := reflect.Indirect(resultValue) + + // create a lookup map from field name (or sqlite:"" tag) + // to the field name + lm := make(map[string]string) + for i := 0; i < target.NumField(); i++ { + fieldType := t.Field(i) + + // skip unexported fields + if !fieldType.IsExported() { + continue + } + + lm[sqlColumnName(fieldType)] = fieldType.Name + } + + // iterate over all columns and assign them to the correct + // fields + for i := 0; i < stmt.ColumnCount(); i++ { + colName := stmt.ColumnName(i) + fieldName, ok := lm[colName] + if !ok { + // there's no target field for this column + // so we can skip it + continue + } + fieldType, _ := t.FieldByName(fieldName) + + value := target.FieldByName(fieldName) + + colType := stmt.ColumnType(i) + + // if the column is reported as NULL we keep + // the field as it is. + // TODO(ppacher): should we set it to nil here? + if colType == sqlite.TypeNull { + continue + } + + // if value is a nil pointer we need to allocate some memory + // first + if getKind(value) == reflect.Ptr && value.IsNil() { + storage := reflect.New(fieldType.Type.Elem()) + + value.Set(storage) + + // make sure value actually points the + // dereferenced target storage + value = storage.Elem() + } + + // execute all decode hooks but make sure we use decodeBasic() as the + // last one. + columnValue, err := runDecodeHooks( + i, + stmt, + fieldType, + value, + append(cfg.DecodeHooks, decodeBasic()), + ) + if err != nil { + return err + } + + // if we don't have a converted value now we try to + // decode basic types + if columnValue == nil { + return fmt.Errorf("cannot decode column %d (type=%s)", i, colType) + } + + //log.Printf("valueTypeName: %s fieldName = %s value-orig = %s value = %s (%v) newValue = %s", value.Type().String(), fieldName, target.FieldByName(fieldName).Type(), value.Type(), value, columnValue) + + // convert it to the target type if conversion is possible + newValue := reflect.ValueOf(columnValue) + if newValue.Type().ConvertibleTo(value.Type()) { + newValue = newValue.Convert(value.Type()) + } + + // assign the new value to the struct field. + value.Set(newValue) + } + + return nil +} + +// DatetimeDecoder is capable of decoding sqlite INTEGER or TEXT storage classes into +// time.Time. For INTEGER storage classes, it supports 'unixnano' struct tag value to +// decide between Unix or UnixNano epoch timestamps. +// +// FIXME(ppacher): update comment about loc parameter and TEXT storage class parsing +// +func DatetimeDecoder(loc *time.Location) DecodeFunc { + return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) { + // we only care about "time.Time" here + if outval.Type().String() != "time.Time" { + return nil, nil + } + + switch stmt.ColumnType(colIdx) { + case sqlite.TypeInteger: + // stored as unix-epoch, if unixnano is set in the struct field tag + // we parse it with nano-second resolution + // TODO(ppacher): actually split the tag value at "," and search + // the slice for "unixnano" + if strings.Contains(fieldDef.Tag.Get("sqlite"), ",unixnano") { + return time.Unix(0, int64(stmt.ColumnInt(colIdx))), nil + } + + return time.Unix(int64(stmt.ColumnInt(colIdx)), 0), nil + + case sqlite.TypeText: + // stored ISO8601 but does not have any timezone information + // assigned so we always treat it as loc here. + t, err := time.ParseInLocation(sqliteTimeFormat, stmt.ColumnText(colIdx), loc) + if err != nil { + return nil, fmt.Errorf("failed to parse %q in %s: %w", stmt.ColumnText(colIdx), fieldDef.Name, err) + } + + return t, nil + + case sqlite.TypeFloat: + // stored as Julian day numbers + return nil, fmt.Errorf("REAL storage type not support for time.Time") + + default: + return nil, fmt.Errorf("unsupported storage type for time.Time: %s", outval.Type()) + } + } +} + +func decodeIntoMap(ctx context.Context, stmt Stmt, mp *map[string]interface{}) error { + if *mp == nil { + *mp = make(map[string]interface{}) + } + + for i := 0; i < stmt.ColumnCount(); i++ { + var x interface{} + val, err := decodeBasic()(i, stmt, reflect.StructField{}, reflect.ValueOf(&x).Elem()) + if err != nil { + return fmt.Errorf("failed to decode column %s: %w", stmt.ColumnName(i), err) + } + + (*mp)[stmt.ColumnName(i)] = val + } + + return nil +} + +func decodeBasic() DecodeFunc { + return func(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value) (interface{}, error) { + valueKind := getKind(outval) + colType := stmt.ColumnType(colIdx) + colName := stmt.ColumnName(colIdx) + + errInvalidType := fmt.Errorf("%w %s for column %s with field type %s", errUnexpectedColumnType, colType.String(), colName, outval.Type()) + + switch valueKind { + case reflect.String: + if colType != sqlite.TypeText { + return nil, errInvalidType + } + return stmt.ColumnText(colIdx), nil + + case reflect.Bool: + // sqlite does not have a BOOL type, it rather stores a 1/0 in a column + // with INTEGER affinity. + if colType != sqlite.TypeInteger { + return nil, errInvalidType + } + return stmt.ColumnBool(colIdx), nil + + case reflect.Float64: + if colType != sqlite.TypeFloat { + return nil, errInvalidType + } + return stmt.ColumnFloat(colIdx), nil + + case reflect.Int, reflect.Uint: // getKind() normalizes all ints to reflect.Int/Uint because sqlite doesn't really care ... + if colType != sqlite.TypeInteger { + return nil, errInvalidType + } + + return stmt.ColumnInt(colIdx), nil + + case reflect.Slice: + if outval.Type().Elem().Kind() != reflect.Uint8 { + return nil, fmt.Errorf("slices other than []byte for BLOB are not supported") + } + + if colType != sqlite.TypeBlob { + return nil, errInvalidType + } + + columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx)) + if err != nil { + return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err) + } + + return columnValue, nil + + case reflect.Interface: + var ( + t reflect.Type + x interface{} + ) + switch colType { + case sqlite.TypeBlob: + t = reflect.TypeOf([]byte{}) + columnValue, err := io.ReadAll(stmt.ColumnReader(colIdx)) + if err != nil { + return nil, fmt.Errorf("failed to read blob for column %s: %w", fieldDef.Name, err) + } + x = columnValue + + case sqlite.TypeFloat: + t = reflect.TypeOf(float64(0)) + x = stmt.ColumnFloat(colIdx) + + case sqlite.TypeInteger: + t = reflect.TypeOf(int(0)) + x = stmt.ColumnInt(colIdx) + + case sqlite.TypeText: + t = reflect.TypeOf(string("")) + x = stmt.ColumnText(colIdx) + + case sqlite.TypeNull: + t = nil + x = nil + + default: + return nil, fmt.Errorf("unsupported column type %s", colType) + } + + if t == nil { + return nil, nil + } + + target := reflect.New(t).Elem() + target.Set(reflect.ValueOf(x)) + + return target.Interface(), nil + + default: + return nil, fmt.Errorf("cannot decode into %s", valueKind) + } + } +} + +func sqlColumnName(fieldType reflect.StructField) string { + tagValue, hasTag := fieldType.Tag.Lookup("sqlite") + if !hasTag { + return fieldType.Name + } + + parts := strings.Split(tagValue, ",") + if parts[0] != "" { + return parts[0] + } + + return fieldType.Name +} + +// runDecodeHooks tries to decode the column value of stmt at index colIdx into outval by running all decode hooks. +// The first hook that returns a non-nil interface wins, other hooks will not be executed. If an error is +// returned by a decode hook runDecodeHooks stops the error is returned to the caller. +func runDecodeHooks(colIdx int, stmt Stmt, fieldDef reflect.StructField, outval reflect.Value, hooks []DecodeFunc) (interface{}, error) { + for _, fn := range hooks { + res, err := fn(colIdx, stmt, fieldDef, outval) + if err != nil { + return res, err + } + + if res != nil { + return res, nil + } + } + + return nil, nil +} + +// getKind returns the kind of value but normalized Int, Uint and Float varaints +// to their base type. +func getKind(val reflect.Value) reflect.Kind { + kind := val.Kind() + return normalizeKind(kind) +} + +func normalizeKind(kind reflect.Kind) reflect.Kind { + switch { + case kind >= reflect.Int && kind <= reflect.Int64: + return reflect.Int + case kind >= reflect.Uint && kind <= reflect.Uint64: + return reflect.Uint + case kind >= reflect.Float32 && kind <= reflect.Float64: + return reflect.Float64 + default: + return kind + } +} + +var DefaultDecodeConfig = DecodeConfig{ + DecodeHooks: []DecodeFunc{ + DatetimeDecoder(time.UTC), + }, +} diff --git a/netquery/orm/decoder_test.go b/netquery/orm/decoder_test.go new file mode 100644 index 00000000..c6c836c0 --- /dev/null +++ b/netquery/orm/decoder_test.go @@ -0,0 +1,475 @@ +package orm + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "zombiezen.com/go/sqlite" +) + +type testStmt struct { + columns []string + values []interface{} + types []sqlite.ColumnType +} + +func (ts testStmt) ColumnCount() int { return len(ts.columns) } +func (ts testStmt) ColumnName(i int) string { return ts.columns[i] } +func (ts testStmt) ColumnBool(i int) bool { return ts.values[i].(bool) } +func (ts testStmt) ColumnText(i int) string { return ts.values[i].(string) } +func (ts testStmt) ColumnFloat(i int) float64 { return ts.values[i].(float64) } +func (ts testStmt) ColumnInt(i int) int { return ts.values[i].(int) } +func (ts testStmt) ColumnReader(i int) *bytes.Reader { return bytes.NewReader(ts.values[i].([]byte)) } +func (ts testStmt) ColumnType(i int) sqlite.ColumnType { return ts.types[i] } + +// compile time check +var _ Stmt = new(testStmt) + +type exampleFieldTypes struct { + S string + I int + F float64 + B bool +} + +type examplePointerTypes struct { + S *string + I *int + F *float64 + B *bool +} + +type exampleStructTags struct { + S string `sqlite:"col_string"` + I int `sqlite:"col_int"` +} + +type exampleIntConv struct { + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int +} + +type exampleBlobTypes struct { + B []byte +} + +type exampleJSONRawTypes struct { + B json.RawMessage +} + +type exampleTimeTypes struct { + T time.Time + TP *time.Time +} + +type exampleInterface struct { + I interface{} + IP *interface{} +} + +func (ett *exampleTimeTypes) Equal(other interface{}) bool { + oett, ok := other.(*exampleTimeTypes) + if !ok { + return false + } + return ett.T.Equal(oett.T) && (ett.TP != nil && oett.TP != nil && ett.TP.Equal(*oett.TP)) || (ett.TP == nil && oett.TP == nil) +} + +type exampleTimeNano struct { + T time.Time `sqlite:",unixnano"` +} + +func (etn *exampleTimeNano) Equal(other interface{}) bool { + oetn, ok := other.(*exampleTimeNano) + if !ok { + return false + } + return etn.T.Equal(oetn.T) +} + +func Test_Decoder(t *testing.T) { + ctx := context.TODO() + + refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC) + + cases := []struct { + Desc string + Stmt testStmt + Result interface{} + Expected interface{} + }{ + { + "Decoding into nil is not allowed", + testStmt{ + columns: nil, + values: nil, + types: nil, + }, + nil, + nil, + }, + { + "Decoding into basic types", + testStmt{ + columns: []string{"S", "I", "F", "B"}, + types: []sqlite.ColumnType{ + sqlite.TypeText, + sqlite.TypeInteger, + sqlite.TypeFloat, + sqlite.TypeInteger, + }, + values: []interface{}{ + "string value", + 1, + 1.2, + true, + }, + }, + &exampleFieldTypes{}, + &exampleFieldTypes{ + S: "string value", + I: 1, + F: 1.2, + B: true, + }, + }, + { + "Decoding into basic types with different order", + testStmt{ + columns: []string{"I", "S", "B", "F"}, + types: []sqlite.ColumnType{ + sqlite.TypeInteger, + sqlite.TypeText, + sqlite.TypeInteger, + sqlite.TypeFloat, + }, + values: []interface{}{ + 1, + "string value", + true, + 1.2, + }, + }, + &exampleFieldTypes{}, + &exampleFieldTypes{ + S: "string value", + I: 1, + F: 1.2, + B: true, + }, + }, + { + "Decoding into basic types with missing values", + testStmt{ + columns: []string{"F", "B"}, + types: []sqlite.ColumnType{ + sqlite.TypeFloat, + sqlite.TypeInteger, + }, + values: []interface{}{ + 1.2, + true, + }, + }, + &exampleFieldTypes{}, + &exampleFieldTypes{ + F: 1.2, + B: true, + }, + }, + { + "Decoding into pointer types", + testStmt{ + columns: []string{"S", "I", "F", "B"}, + types: []sqlite.ColumnType{ + sqlite.TypeText, + sqlite.TypeInteger, + sqlite.TypeFloat, + sqlite.TypeInteger, + }, + values: []interface{}{ + "string value", + 1, + 1.2, + true, + }, + }, + &examplePointerTypes{}, + func() interface{} { + s := "string value" + i := 1 + f := 1.2 + b := true + + return &examplePointerTypes{ + S: &s, + I: &i, + F: &f, + B: &b, + } + }, + }, + { + "Decoding into pointer types with missing values", + testStmt{ + columns: []string{"S", "B"}, + types: []sqlite.ColumnType{ + sqlite.TypeText, + sqlite.TypeInteger, + sqlite.TypeFloat, + sqlite.TypeInteger, + }, + values: []interface{}{ + "string value", + true, + }, + }, + &examplePointerTypes{}, + func() interface{} { + s := "string value" + b := true + + return &examplePointerTypes{ + S: &s, + B: &b, + } + }, + }, + { + "Decoding into fields with struct tags", + testStmt{ + columns: []string{"col_string", "col_int"}, + types: []sqlite.ColumnType{ + sqlite.TypeText, + sqlite.TypeInteger, + }, + values: []interface{}{ + "string value", + 1, + }, + }, + &exampleStructTags{}, + &exampleStructTags{ + S: "string value", + I: 1, + }, + }, + { + "Decoding into correct int type", + testStmt{ + columns: []string{"I8", "I16", "I32", "I64", "I"}, + types: []sqlite.ColumnType{ + sqlite.TypeInteger, + sqlite.TypeInteger, + sqlite.TypeInteger, + sqlite.TypeInteger, + sqlite.TypeInteger, + }, + values: []interface{}{ + 1, + 1, + 1, + 1, + 1, + }, + }, + &exampleIntConv{}, + &exampleIntConv{ + 1, 1, 1, 1, 1, + }, + }, + { + "Handling NULL values for basic types", + testStmt{ + columns: []string{"S", "I", "F"}, + types: []sqlite.ColumnType{ + sqlite.TypeNull, + sqlite.TypeNull, + sqlite.TypeFloat, + }, + values: []interface{}{ + // we use nil here but actually that does not matter + nil, + nil, + 1.0, + }, + }, + &exampleFieldTypes{}, + &exampleFieldTypes{ + F: 1.0, + }, + }, + { + "Handling NULL values for pointer types", + testStmt{ + columns: []string{"S", "I", "F"}, + types: []sqlite.ColumnType{ + sqlite.TypeNull, + sqlite.TypeNull, + sqlite.TypeFloat, + }, + values: []interface{}{ + // we use nil here but actually that does not matter + nil, + nil, + 1.0, + }, + }, + &examplePointerTypes{}, + func() interface{} { + f := 1.0 + + return &examplePointerTypes{F: &f} + }, + }, + { + "Handling blob types", + testStmt{ + columns: []string{"B"}, + types: []sqlite.ColumnType{ + sqlite.TypeBlob, + }, + values: []interface{}{ + ([]byte)("hello world"), + }, + }, + &exampleBlobTypes{}, + &exampleBlobTypes{ + B: ([]byte)("hello world"), + }, + }, + { + "Handling blob types as json.RawMessage", + testStmt{ + columns: []string{"B"}, + types: []sqlite.ColumnType{ + sqlite.TypeBlob, + }, + values: []interface{}{ + ([]byte)("hello world"), + }, + }, + &exampleJSONRawTypes{}, + &exampleJSONRawTypes{ + B: (json.RawMessage)("hello world"), + }, + }, + { + "Handling time.Time and pointers to it", + testStmt{ + columns: []string{"T", "TP"}, + types: []sqlite.ColumnType{ + sqlite.TypeInteger, + sqlite.TypeInteger, + }, + values: []interface{}{ + int(refTime.Unix()), + int(refTime.Unix()), + }, + }, + &exampleTimeTypes{}, + &exampleTimeTypes{ + T: refTime, + TP: &refTime, + }, + }, + { + "Handling time.Time in nano-second resolution (struct tags)", + testStmt{ + columns: []string{"T", "TP"}, + types: []sqlite.ColumnType{ + sqlite.TypeInteger, + sqlite.TypeInteger, + }, + values: []interface{}{ + int(refTime.UnixNano()), + int(refTime.UnixNano()), + }, + }, + &exampleTimeNano{}, + &exampleTimeNano{ + T: refTime, + }, + }, + { + "Decoding into interface", + testStmt{ + columns: []string{"I", "IP"}, + types: []sqlite.ColumnType{ + sqlite.TypeText, + sqlite.TypeText, + }, + values: []interface{}{ + "value1", + "value2", + }, + }, + &exampleInterface{}, + func() interface{} { + var x interface{} + x = "value2" + + return &exampleInterface{ + I: "value1", + IP: &x, + } + }, + }, + { + "Decoding into map[string]interface{}", + testStmt{ + columns: []string{"I", "F", "S", "B"}, + types: []sqlite.ColumnType{ + sqlite.TypeInteger, + sqlite.TypeFloat, + sqlite.TypeText, + sqlite.TypeBlob, + }, + values: []interface{}{ + 1, + 1.1, + "string value", + []byte("blob value"), + }, + }, + new(map[string]interface{}), + &map[string]interface{}{ + "I": 1, + "F": 1.1, + "S": "string value", + "B": []byte("blob value"), + }, + }, + } + + for idx := range cases { + c := cases[idx] + t.Run(c.Desc, func(t *testing.T) { + t.Parallel() + + err := DecodeStmt(ctx, c.Stmt, c.Result, DefaultDecodeConfig) + if fn, ok := c.Expected.(func() interface{}); ok { + c.Expected = fn() + } + + if c.Expected == nil { + assert.Error(t, err, c.Desc) + } else { + assert.NoError(t, err, c.Desc) + + if equaler, ok := c.Expected.(interface{ Equal(x interface{}) bool }); ok { + assert.True(t, equaler.Equal(c.Result)) + } else { + assert.Equal(t, c.Expected, c.Result) + } + } + }) + } +} diff --git a/netquery/orm/encoder.go b/netquery/orm/encoder.go new file mode 100644 index 00000000..c2d9a62c --- /dev/null +++ b/netquery/orm/encoder.go @@ -0,0 +1,124 @@ +package orm + +import ( + "context" + "fmt" + "reflect" + "time" + + "zombiezen.com/go/sqlite" +) + +type ( + EncodeFunc func(col *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) + + EncodeConfig struct { + EncodeHooks []EncodeFunc + } +) + +// EncodeAsMap returns a map that contains the value of each struct field of +// r using the sqlite column name as a map key. It either uses the name of the +// exported struct field or the value of the "sqlite" tag. +func EncodeAsMap(ctx context.Context, r interface{}, keyPrefix string, cfg EncodeConfig) (map[string]interface{}, error) { + // make sure we work on a struct type + val := reflect.Indirect(reflect.ValueOf(r)) + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("%w, got %T", errStructExpected, r) + } + + res := make(map[string]interface{}, val.NumField()) + + for i := 0; i < val.NumField(); i++ { + fieldType := val.Type().Field(i) + field := val.Field(i) + + // skip unexported fields + if !fieldType.IsExported() { + continue + } + + colDev, err := getColumnDef(fieldType) + if err != nil { + return nil, fmt.Errorf("failed to get column definition for %s: %w", fieldType.Name, err) + } + + x, found, err := runEncodeHooks(colDev, fieldType.Type, field, cfg.EncodeHooks) + if err != nil { + return nil, fmt.Errorf("failed to run encode hooks: %w", err) + } + + if !found { + if reflect.Indirect(field).IsValid() { + x = reflect.Indirect(field).Interface() + } + } + + res[keyPrefix+sqlColumnName(fieldType)] = x + + } + + return res, nil +} + +func DatetimeEncoder(loc *time.Location) EncodeFunc { + return func(colDev *ColumnDef, valType reflect.Type, val reflect.Value) (interface{}, bool, error) { + // if fieldType holds a pointer we need to dereference the value + ft := valType.String() + if valType.Kind() == reflect.Ptr { + ft = valType.Elem().String() + val = reflect.Indirect(val) + } + + // we only care about "time.Time" here + if ft != "time.Time" { + return nil, false, nil + } + + // handle the zero time as a NULL. + if !val.IsValid() || val.IsZero() { + return nil, true, nil + } + + valInterface := val.Interface() + t, ok := valInterface.(time.Time) + if !ok { + return nil, false, fmt.Errorf("cannot convert reflect value to time.Time") + } + + switch colDev.Type { + case sqlite.TypeInteger: + if colDev.UnixNano { + return t.UnixNano(), true, nil + } + return t.Unix(), true, nil + case sqlite.TypeText: + str := t.In(loc).Format(sqliteTimeFormat) + + return str, true, nil + } + + return nil, false, fmt.Errorf("cannot store time.Time in %s", colDev.Type) + } +} + +func runEncodeHooks(colDev *ColumnDef, valType reflect.Type, val reflect.Value, hooks []EncodeFunc) (interface{}, bool, error) { + for _, fn := range hooks { + res, end, err := fn(colDev, valType, val) + if err != nil { + return res, false, err + } + + if end { + return res, true, nil + } + } + + return nil, false, nil +} + +var DefaultEncodeConfig = EncodeConfig{ + EncodeHooks: []EncodeFunc{ + DatetimeEncoder(time.UTC), + }, +} diff --git a/netquery/orm/encoder_test.go b/netquery/orm/encoder_test.go new file mode 100644 index 00000000..056bf953 --- /dev/null +++ b/netquery/orm/encoder_test.go @@ -0,0 +1,126 @@ +package orm + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_EncodeAsMap(t *testing.T) { + ctx := context.TODO() + refTime := time.Date(2022, time.February, 15, 9, 51, 00, 00, time.UTC) + + cases := []struct { + Desc string + Input interface{} + Expected map[string]interface{} + }{ + { + "Encode basic types", + struct { + I int + F float64 + S string + B []byte + }{ + I: 1, + F: 1.2, + S: "string", + B: ([]byte)("bytes"), + }, + map[string]interface{}{ + "I": 1, + "F": 1.2, + "S": "string", + "B": ([]byte)("bytes"), + }, + }, + { + "Encode using struct tags", + struct { + I int `sqlite:"col_int"` + S string `sqlite:"col_string"` + }{ + I: 1, + S: "string value", + }, + map[string]interface{}{ + "col_int": 1, + "col_string": "string value", + }, + }, + { + "Ignore Private fields", + struct { + I int + s string + }{ + I: 1, + s: "string value", + }, + map[string]interface{}{ + "I": 1, + }, + }, + { + "Handle Pointers", + struct { + I *int + S *string + }{ + I: new(int), + }, + map[string]interface{}{ + "I": 0, + "S": nil, + }, + }, + { + "Handle time.Time types", + struct { + TinInt time.Time `sqlite:",integer,unixnano"` + TinString time.Time `sqlite:",text"` + }{ + TinInt: refTime, + TinString: refTime, + }, + map[string]interface{}{ + "TinInt": refTime.UnixNano(), + "TinString": refTime.Format(sqliteTimeFormat), + }, + }, + { + "Handle time.Time pointer types", + struct { + TinInt *time.Time `sqlite:",integer,unixnano"` + TinString *time.Time `sqlite:",text"` + Tnil1 *time.Time `sqlite:",text"` + Tnil2 *time.Time `sqlite:",text"` + }{ + TinInt: &refTime, + TinString: &refTime, + Tnil1: nil, + Tnil2: (*time.Time)(nil), + }, + map[string]interface{}{ + "TinInt": refTime.UnixNano(), + "TinString": refTime.Format(sqliteTimeFormat), + "Tnil1": nil, + "Tnil2": nil, + }, + }, + } + + for idx := range cases { + c := cases[idx] + t.Run(c.Desc, func(t *testing.T) { + // t.Parallel() + + res, err := EncodeAsMap(ctx, c.Input, "", DefaultEncodeConfig) + assert.NoError(t, err) + assert.Equal(t, c.Expected, res) + }) + } +} diff --git a/netquery/orm/query_runner.go b/netquery/orm/query_runner.go new file mode 100644 index 00000000..d94e62db --- /dev/null +++ b/netquery/orm/query_runner.go @@ -0,0 +1,123 @@ +package orm + +import ( + "context" + "fmt" + "reflect" + + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +type ( + QueryOption func(opts *queryOpts) + + queryOpts struct { + Transient bool + Args []interface{} + NamedArgs map[string]interface{} + Result interface{} + DecodeConfig DecodeConfig + } +) + +func WithTransient() QueryOption { + return func(opts *queryOpts) { + opts.Transient = true + } +} + +func WithArgs(args ...interface{}) QueryOption { + return func(opts *queryOpts) { + opts.Args = args + } +} + +func WithNamedArgs(args map[string]interface{}) QueryOption { + return func(opts *queryOpts) { + opts.NamedArgs = args + } +} + +func WithResult(result interface{}) QueryOption { + return func(opts *queryOpts) { + opts.Result = result + } +} + +func WithDecodeConfig(cfg DecodeConfig) QueryOption { + return func(opts *queryOpts) { + opts.DecodeConfig = cfg + } +} + +func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...QueryOption) error { + args := queryOpts{ + DecodeConfig: DefaultDecodeConfig, + } + + for _, fn := range modifiers { + fn(&args) + } + + opts := &sqlitex.ExecOptions{ + Args: args.Args, + Named: args.NamedArgs, + } + + var ( + sliceVal reflect.Value + valElemType reflect.Type + ) + + if args.Result != nil { + target := args.Result + outVal := reflect.ValueOf(target) + if outVal.Kind() != reflect.Ptr { + return fmt.Errorf("target must be a pointer, got %T", target) + } + + sliceVal = reflect.Indirect(outVal) + if !sliceVal.IsValid() || sliceVal.IsNil() { + newVal := reflect.Zero(outVal.Type().Elem()) + sliceVal.Set(newVal) + } + + kind := sliceVal.Kind() + if kind != reflect.Slice { + return fmt.Errorf("target but be pointer to slice, got %T", target) + } + valType := sliceVal.Type() + valElemType = valType.Elem() + + opts.ResultFunc = func(stmt *sqlite.Stmt) error { + var currentField reflect.Value + + currentField = reflect.New(valElemType) + + if err := DecodeStmt(ctx, stmt, currentField.Interface(), args.DecodeConfig); err != nil { + return err + } + + sliceVal = reflect.Append(sliceVal, reflect.Indirect(currentField)) + + return nil + } + } + + var err error + if args.Transient { + err = sqlitex.ExecuteTransient(conn, sql, opts) + } else { + err = sqlitex.Execute(conn, sql, opts) + } + if err != nil { + return err + } + + if args.Result != nil { + reflect.Indirect(reflect.ValueOf(args.Result)).Set(sliceVal) + } + + return nil +} diff --git a/netquery/orm/schema_builder.go b/netquery/orm/schema_builder.go new file mode 100644 index 00000000..aa57e21a --- /dev/null +++ b/netquery/orm/schema_builder.go @@ -0,0 +1,220 @@ +package orm + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "zombiezen.com/go/sqlite" +) + +var ( + errSkipStructField = errors.New("struct field should be skipped") +) + +var ( + TagUnixNano = "unixnano" + TagPrimaryKey = "primary" + TagAutoIncrement = "autoincrement" + TagNotNull = "not-null" + TagNullable = "nullable" + TagTypeInt = "integer" + TagTypeText = "text" + TagTypePrefixVarchar = "varchar" + TagTypeBlob = "blob" + TagTypeFloat = "float" +) + +var sqlTypeMap = map[sqlite.ColumnType]string{ + sqlite.TypeBlob: "BLOB", + sqlite.TypeFloat: "REAL", + sqlite.TypeInteger: "INTEGER", + sqlite.TypeText: "TEXT", +} + +type ( + TableSchema struct { + Name string + Columns []ColumnDef + } + + ColumnDef struct { + Name string + Nullable bool + Type sqlite.ColumnType + Length int + PrimaryKey bool + AutoIncrement bool + UnixNano bool + } +) + +func (ts TableSchema) CreateStatement(ifNotExists bool) string { + sql := "CREATE TABLE" + if ifNotExists { + sql += " IF NOT EXISTS" + } + sql += " " + ts.Name + " ( " + + for idx, col := range ts.Columns { + sql += col.AsSQL() + if idx < len(ts.Columns)-1 { + sql += ", " + } + } + + sql += " );" + return sql +} + +func (def ColumnDef) AsSQL() string { + sql := def.Name + " " + + if def.Type == sqlite.TypeText && def.Length > 0 { + sql += fmt.Sprintf("VARCHAR(%d)", def.Length) + } else { + sql += sqlTypeMap[def.Type] + } + + if def.PrimaryKey { + sql += " PRIMARY KEY" + } + if def.AutoIncrement { + sql += " AUTOINCREMENT" + } + if !def.Nullable { + sql += " NOT NULL" + } + + return sql +} + +func GenerateTableSchema(name string, d interface{}) (*TableSchema, error) { + ts := &TableSchema{ + Name: name, + } + + val := reflect.Indirect(reflect.ValueOf(d)) + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("%w, got %T", errStructExpected, d) + } + + for i := 0; i < val.NumField(); i++ { + fieldType := val.Type().Field(i) + if !fieldType.IsExported() { + continue + } + + def, err := getColumnDef(fieldType) + if err != nil { + if errors.Is(err, errSkipStructField) { + continue + } + + return nil, fmt.Errorf("struct field %s: %w", fieldType.Name, err) + } + + ts.Columns = append(ts.Columns, *def) + } + + return ts, nil +} + +func getColumnDef(fieldType reflect.StructField) (*ColumnDef, error) { + def := &ColumnDef{ + Name: fieldType.Name, + Nullable: fieldType.Type.Kind() == reflect.Ptr, + } + + ft := fieldType.Type + + if fieldType.Type.Kind() == reflect.Ptr { + ft = fieldType.Type.Elem() + } + + kind := normalizeKind(ft.Kind()) + + switch kind { + case reflect.Int: + def.Type = sqlite.TypeInteger + + case reflect.Float64: + def.Type = sqlite.TypeFloat + + case reflect.String: + def.Type = sqlite.TypeText + + case reflect.Slice: + // only []byte/[]uint8 is supported + if ft.Elem().Kind() != reflect.Uint8 { + return nil, fmt.Errorf("slices of type %s is not supported", ft.Elem()) + } + + def.Type = sqlite.TypeBlob + } + + if err := applyStructFieldTag(fieldType, def); err != nil { + return nil, err + } + + return def, nil +} + +// applyStructFieldTag parses the sqlite:"" struct field tag and update the column +// definition def accordingly. +func applyStructFieldTag(fieldType reflect.StructField, def *ColumnDef) error { + parts := strings.Split(fieldType.Tag.Get("sqlite"), ",") + if len(parts) > 0 && parts[0] != "" { + if parts[0] == "-" { + return errSkipStructField + } + + def.Name = parts[0] + } + + if len(parts) > 1 { + for _, k := range parts[1:] { + switch k { + // column modifieres + case TagPrimaryKey: + def.PrimaryKey = true + case TagAutoIncrement: + def.AutoIncrement = true + case TagNotNull: + def.Nullable = false + case TagNullable: + def.Nullable = true + case TagUnixNano: + def.UnixNano = true + + // basic column types + case TagTypeInt: + def.Type = sqlite.TypeInteger + case TagTypeText: + def.Type = sqlite.TypeText + case TagTypeFloat: + def.Type = sqlite.TypeFloat + case TagTypeBlob: + def.Type = sqlite.TypeBlob + + // advanced column types + default: + if strings.HasPrefix(k, TagTypePrefixVarchar) { + lenStr := strings.TrimSuffix(strings.TrimPrefix(k, TagTypePrefixVarchar+"("), ")") + length, err := strconv.ParseInt(lenStr, 10, 0) + if err != nil { + return fmt.Errorf("failed to parse varchar length %q: %w", lenStr, err) + } + + def.Type = sqlite.TypeText + def.Length = int(length) + } + + } + } + } + + return nil +} diff --git a/netquery/orm/schema_builder_test.go b/netquery/orm/schema_builder_test.go new file mode 100644 index 00000000..7012076d --- /dev/null +++ b/netquery/orm/schema_builder_test.go @@ -0,0 +1,41 @@ +package orm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_SchemaBuilder(t *testing.T) { + cases := []struct { + Name string + Model interface{} + ExpectedSQL string + }{ + { + "Simple", + struct { + ID int `sqlite:"id,primary,autoincrement"` + Text string `sqlite:"text,nullable"` + Int *int `sqlite:",not-null"` + Float interface{} `sqlite:",float,nullable"` + }{}, + `CREATE TABLE Simple ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, text TEXT, Int INTEGER NOT NULL, Float REAL );`, + }, + { + "Varchar", + struct { + S string `sqlite:",varchar(10)"` + }{}, + `CREATE TABLE Varchar ( S VARCHAR(10) NOT NULL );`, + }, + } + + for idx := range cases { + c := cases[idx] + + res, err := GenerateTableSchema(c.Name, c.Model) + assert.NoError(t, err) + assert.Equal(t, c.ExpectedSQL, res.CreateStatement(false)) + } +}