Restructure modules (#1572)

* Move portbase into monorepo

* Add new simple module mgr

* [WIP] Switch to new simple module mgr

* Add StateMgr and more worker variants

* [WIP] Switch more modules

* [WIP] Switch more modules

* [WIP] swtich more modules

* [WIP] switch all SPN modules

* [WIP] switch all service modules

* [WIP] Convert all workers to the new module system

* [WIP] add new task system to module manager

* [WIP] Add second take for scheduling workers

* [WIP] Add FIXME for bugs in new scheduler

* [WIP] Add minor improvements to scheduler

* [WIP] Add new worker scheduler

* [WIP] Fix more bug related to new module system

* [WIP] Fix start handing of the new module system

* [WIP] Improve startup process

* [WIP] Fix minor issues

* [WIP] Fix missing subsystem in settings

* [WIP] Initialize managers in constructor

* [WIP] Move module event initialization to constrictors

* [WIP] Fix setting for enabling and disabling the SPN module

* [WIP] Move API registeration into module construction

* [WIP] Update states mgr for all modules

* [WIP] Add CmdLine operation support

* Add state helper methods to module group and instance

* Add notification and module status handling to status package

* Fix starting issues

* Remove pilot widget and update security lock to new status data

* Remove debug logs

* Improve http server shutdown

* Add workaround for cleanly shutting down firewall+netquery

* Improve logging

* Add syncing states with notifications for new module system

* Improve starting, stopping, shutdown; resolve FIXMEs/TODOs

* [WIP] Fix most unit tests

* Review new module system and fix minor issues

* Push shutdown and restart events again via API

* Set sleep mode via interface

* Update example/template module

* [WIP] Fix spn/cabin unit test

* Remove deprecated UI elements

* Make log output more similar for the logging transition phase

* Switch spn hub and observer cmds to new module system

* Fix log sources

* Make worker mgr less error prone

* Fix tests and minor issues

* Fix observation hub

* Improve shutdown and restart handling

* Split up big connection.go source file

* Move varint and dsd packages to structures repo

* Improve expansion test

* Fix linter warnings

* Fix interception module on windows

* Fix linter errors

---------

Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
Daniel Hååvi
2024-08-09 17:15:48 +02:00
committed by GitHub
parent 10a77498f4
commit 80664d1a27
647 changed files with 37690 additions and 3366 deletions

View File

@@ -0,0 +1,55 @@
# Query
## Control Flow
- Grouping with `(` and `)`
- Chaining with `and` and `or`
- _NO_ mixing! Be explicit and use grouping.
- Negation with `not`
- in front of expression for group: `not (...)`
- inside expression for clause: `name not matches "^King "`
## Selectors
Supported by all feeders:
- root level field: `field`
- sub level field: `field.sub`
- array/slice/map access: `map.0`
- array/slice/map length: `map.#`
Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys.
## Operators
| Name | Textual | Req. Type | Internal Type | Compared with |
|-------------------------|--------------------|-----------|---------------|---------------------------|
| Equals | `==` | int | int64 | `==` |
| GreaterThan | `>` | int | int64 | `>` |
| GreaterThanOrEqual | `>=` | int | int64 | `>=` |
| LessThan | `<` | int | int64 | `<` |
| LessThanOrEqual | `<=` | int | int64 | `<=` |
| FloatEquals | `f==` | float | float64 | `==` |
| FloatGreaterThan | `f>` | float | float64 | `>` |
| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` |
| FloatLessThan | `f<` | float | float64 | `<` |
| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` |
| SameAs | `sameas`, `s==` | string | string | `==` |
| Contains | `contains`, `co` | string | string | `strings.Contains()` |
| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` |
| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` |
| In | `in` | string | string | for loop with `==` |
| Matches | `matches`, `re` | string | string | `regexp.Regexp.Matches()` |
| Is | `is` | bool* | bool | `==` |
| Exists | `exists`, `ex` | any | n/a | n/a |
\*accepts strings: 1, t, T, true, True, TRUE, 0, f, F, false, False, FALSE
## Escaping
If you need to use a control character within a value (ie. not for controlling), escape it with `\`.
It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible.
| Location | Characters to be escaped |
|---|---|
| Within parenthesis (`"`) | `"`, `\` |
| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) |

View File

@@ -0,0 +1,46 @@
package query
import (
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
)
// And combines multiple conditions with a logical _AND_ operator.
func And(conditions ...Condition) Condition {
return &andCond{
conditions: conditions,
}
}
type andCond struct {
conditions []Condition
}
func (c *andCond) complies(acc accessor.Accessor) bool {
for _, cond := range c.conditions {
if !cond.complies(acc) {
return false
}
}
return true
}
func (c *andCond) check() (err error) {
for _, cond := range c.conditions {
err = cond.check()
if err != nil {
return err
}
}
return nil
}
func (c *andCond) string() string {
all := make([]string, 0, len(c.conditions))
for _, cond := range c.conditions {
all = append(all, cond.string())
}
return fmt.Sprintf("(%s)", strings.Join(all, " and "))
}

View File

@@ -0,0 +1,69 @@
package query
import (
"errors"
"fmt"
"strconv"
"github.com/safing/portmaster/base/database/accessor"
)
type boolCondition struct {
key string
operator uint8
value bool
}
func newBoolCondition(key string, operator uint8, value interface{}) *boolCondition {
var parsedValue bool
switch v := value.(type) {
case bool:
parsedValue = v
case string:
var err error
parsedValue, err = strconv.ParseBool(v)
if err != nil {
return &boolCondition{
key: fmt.Sprintf("could not parse \"%s\" to bool: %s", v, err),
operator: errorPresent,
}
}
default:
return &boolCondition{
key: fmt.Sprintf("incompatible value %v for int64", value),
operator: errorPresent,
}
}
return &boolCondition{
key: key,
operator: operator,
value: parsedValue,
}
}
func (c *boolCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetBool(c.key)
if !ok {
return false
}
switch c.operator {
case Is:
return comp == c.value
default:
return false
}
}
func (c *boolCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *boolCondition) string() string {
return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value)
}

View File

@@ -0,0 +1,27 @@
package query
import (
"github.com/safing/portmaster/base/database/accessor"
)
type errorCondition struct {
err error
}
func newErrorCondition(err error) *errorCondition {
return &errorCondition{
err: err,
}
}
func (c *errorCondition) complies(acc accessor.Accessor) bool {
return false
}
func (c *errorCondition) check() error {
return c.err
}
func (c *errorCondition) string() string {
return "[ERROR]"
}

View File

@@ -0,0 +1,35 @@
package query
import (
"errors"
"fmt"
"github.com/safing/portmaster/base/database/accessor"
)
type existsCondition struct {
key string
operator uint8
}
func newExistsCondition(key string, operator uint8) *existsCondition {
return &existsCondition{
key: key,
operator: operator,
}
}
func (c *existsCondition) complies(acc accessor.Accessor) bool {
return acc.Exists(c.key)
}
func (c *existsCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *existsCondition) string() string {
return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator))
}

View File

@@ -0,0 +1,97 @@
package query
import (
"errors"
"fmt"
"strconv"
"github.com/safing/portmaster/base/database/accessor"
)
type floatCondition struct {
key string
operator uint8
value float64
}
func newFloatCondition(key string, operator uint8, value interface{}) *floatCondition {
var parsedValue float64
switch v := value.(type) {
case int:
parsedValue = float64(v)
case int8:
parsedValue = float64(v)
case int16:
parsedValue = float64(v)
case int32:
parsedValue = float64(v)
case int64:
parsedValue = float64(v)
case uint:
parsedValue = float64(v)
case uint8:
parsedValue = float64(v)
case uint16:
parsedValue = float64(v)
case uint32:
parsedValue = float64(v)
case float32:
parsedValue = float64(v)
case float64:
parsedValue = v
case string:
var err error
parsedValue, err = strconv.ParseFloat(v, 64)
if err != nil {
return &floatCondition{
key: fmt.Sprintf("could not parse %s to float64: %s", v, err),
operator: errorPresent,
}
}
default:
return &floatCondition{
key: fmt.Sprintf("incompatible value %v for float64", value),
operator: errorPresent,
}
}
return &floatCondition{
key: key,
operator: operator,
value: parsedValue,
}
}
func (c *floatCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetFloat(c.key)
if !ok {
return false
}
switch c.operator {
case FloatEquals:
return comp == c.value
case FloatGreaterThan:
return comp > c.value
case FloatGreaterThanOrEqual:
return comp >= c.value
case FloatLessThan:
return comp < c.value
case FloatLessThanOrEqual:
return comp <= c.value
default:
return false
}
}
func (c *floatCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *floatCondition) string() string {
return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value)
}

View File

@@ -0,0 +1,93 @@
package query
import (
"errors"
"fmt"
"strconv"
"github.com/safing/portmaster/base/database/accessor"
)
type intCondition struct {
key string
operator uint8
value int64
}
func newIntCondition(key string, operator uint8, value interface{}) *intCondition {
var parsedValue int64
switch v := value.(type) {
case int:
parsedValue = int64(v)
case int8:
parsedValue = int64(v)
case int16:
parsedValue = int64(v)
case int32:
parsedValue = int64(v)
case int64:
parsedValue = v
case uint:
parsedValue = int64(v)
case uint8:
parsedValue = int64(v)
case uint16:
parsedValue = int64(v)
case uint32:
parsedValue = int64(v)
case string:
var err error
parsedValue, err = strconv.ParseInt(v, 10, 64)
if err != nil {
return &intCondition{
key: fmt.Sprintf("could not parse %s to int64: %s (hint: use \"sameas\" to compare strings)", v, err),
operator: errorPresent,
}
}
default:
return &intCondition{
key: fmt.Sprintf("incompatible value %v for int64", value),
operator: errorPresent,
}
}
return &intCondition{
key: key,
operator: operator,
value: parsedValue,
}
}
func (c *intCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetInt(c.key)
if !ok {
return false
}
switch c.operator {
case Equals:
return comp == c.value
case GreaterThan:
return comp > c.value
case GreaterThanOrEqual:
return comp >= c.value
case LessThan:
return comp < c.value
case LessThanOrEqual:
return comp <= c.value
default:
return false
}
}
func (c *intCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *intCondition) string() string {
return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value)
}

View File

@@ -0,0 +1,36 @@
package query
import (
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
)
// Not negates the supplied condition.
func Not(c Condition) Condition {
return &notCond{
notC: c,
}
}
type notCond struct {
notC Condition
}
func (c *notCond) complies(acc accessor.Accessor) bool {
return !c.notC.complies(acc)
}
func (c *notCond) check() error {
return c.notC.check()
}
func (c *notCond) string() string {
next := c.notC.string()
if strings.HasPrefix(next, "(") {
return fmt.Sprintf("not %s", c.notC.string())
}
splitted := strings.Split(next, " ")
return strings.Join(append([]string{splitted[0], "not"}, splitted[1:]...), " ")
}

View File

@@ -0,0 +1,46 @@
package query
import (
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
)
// Or combines multiple conditions with a logical _OR_ operator.
func Or(conditions ...Condition) Condition {
return &orCond{
conditions: conditions,
}
}
type orCond struct {
conditions []Condition
}
func (c *orCond) complies(acc accessor.Accessor) bool {
for _, cond := range c.conditions {
if cond.complies(acc) {
return true
}
}
return false
}
func (c *orCond) check() (err error) {
for _, cond := range c.conditions {
err = cond.check()
if err != nil {
return err
}
}
return nil
}
func (c *orCond) string() string {
all := make([]string, 0, len(c.conditions))
for _, cond := range c.conditions {
all = append(all, cond.string())
}
return fmt.Sprintf("(%s)", strings.Join(all, " or "))
}

View File

@@ -0,0 +1,63 @@
package query
import (
"errors"
"fmt"
"regexp"
"github.com/safing/portmaster/base/database/accessor"
)
type regexCondition struct {
key string
operator uint8
regex *regexp.Regexp
}
func newRegexCondition(key string, operator uint8, value interface{}) *regexCondition {
switch v := value.(type) {
case string:
r, err := regexp.Compile(v)
if err != nil {
return &regexCondition{
key: fmt.Sprintf("could not compile regex \"%s\": %s", v, err),
operator: errorPresent,
}
}
return &regexCondition{
key: key,
operator: operator,
regex: r,
}
default:
return &regexCondition{
key: fmt.Sprintf("incompatible value %v for string", value),
operator: errorPresent,
}
}
}
func (c *regexCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}
switch c.operator {
case Matches:
return c.regex.MatchString(comp)
default:
return false
}
}
func (c *regexCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *regexCondition) string() string {
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String()))
}

View File

@@ -0,0 +1,62 @@
package query
import (
"errors"
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
)
type stringCondition struct {
key string
operator uint8
value string
}
func newStringCondition(key string, operator uint8, value interface{}) *stringCondition {
switch v := value.(type) {
case string:
return &stringCondition{
key: key,
operator: operator,
value: v,
}
default:
return &stringCondition{
key: fmt.Sprintf("incompatible value %v for string", value),
operator: errorPresent,
}
}
}
func (c *stringCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}
switch c.operator {
case SameAs:
return c.value == comp
case Contains:
return strings.Contains(comp, c.value)
case StartsWith:
return strings.HasPrefix(comp, c.value)
case EndsWith:
return strings.HasSuffix(comp, c.value)
default:
return false
}
}
func (c *stringCondition) check() error {
if c.operator == errorPresent {
return errors.New(c.key)
}
return nil
}
func (c *stringCondition) string() string {
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value))
}

View File

@@ -0,0 +1,69 @@
package query
import (
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
"github.com/safing/portmaster/base/utils"
)
type stringSliceCondition struct {
key string
operator uint8
value []string
}
func newStringSliceCondition(key string, operator uint8, value interface{}) *stringSliceCondition {
switch v := value.(type) {
case string:
parsedValue := strings.Split(v, ",")
if len(parsedValue) < 2 {
return &stringSliceCondition{
key: v,
operator: errorPresent,
}
}
return &stringSliceCondition{
key: key,
operator: operator,
value: parsedValue,
}
case []string:
return &stringSliceCondition{
key: key,
operator: operator,
value: v,
}
default:
return &stringSliceCondition{
key: fmt.Sprintf("incompatible value %v for []string", value),
operator: errorPresent,
}
}
}
func (c *stringSliceCondition) complies(acc accessor.Accessor) bool {
comp, ok := acc.GetString(c.key)
if !ok {
return false
}
switch c.operator {
case In:
return utils.StringInSlice(c.value, comp)
default:
return false
}
}
func (c *stringSliceCondition) check() error {
if c.operator == errorPresent {
return fmt.Errorf("could not parse \"%s\" to []string", c.key)
}
return nil
}
func (c *stringSliceCondition) string() string {
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ",")))
}

View File

@@ -0,0 +1,71 @@
package query
import (
"fmt"
"github.com/safing/portmaster/base/database/accessor"
)
// Condition is an interface to provide a common api to all condition types.
type Condition interface {
complies(acc accessor.Accessor) bool
check() error
string() string
}
// Operators.
const (
Equals uint8 = iota // int
GreaterThan // int
GreaterThanOrEqual // int
LessThan // int
LessThanOrEqual // int
FloatEquals // float
FloatGreaterThan // float
FloatGreaterThanOrEqual // float
FloatLessThan // float
FloatLessThanOrEqual // float
SameAs // string
Contains // string
StartsWith // string
EndsWith // string
In // stringSlice
Matches // regex
Is // bool: accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE
Exists // any
errorPresent uint8 = 255
)
// Where returns a condition to add to a query.
func Where(key string, operator uint8, value interface{}) Condition {
switch operator {
case Equals,
GreaterThan,
GreaterThanOrEqual,
LessThan,
LessThanOrEqual:
return newIntCondition(key, operator, value)
case FloatEquals,
FloatGreaterThan,
FloatGreaterThanOrEqual,
FloatLessThan,
FloatLessThanOrEqual:
return newFloatCondition(key, operator, value)
case SameAs,
Contains,
StartsWith,
EndsWith:
return newStringCondition(key, operator, value)
case In:
return newStringSliceCondition(key, operator, value)
case Matches:
return newRegexCondition(key, operator, value)
case Is:
return newBoolCondition(key, operator, value)
case Exists:
return newExistsCondition(key, operator)
default:
return newErrorCondition(fmt.Errorf("no operator with ID %d", operator))
}
}

View File

@@ -0,0 +1,86 @@
package query
import "testing"
func testSuccess(t *testing.T, c Condition) {
t.Helper()
err := c.check()
if err != nil {
t.Errorf("failed: %s", err)
}
}
func TestInterfaces(t *testing.T) {
t.Parallel()
testSuccess(t, newIntCondition("banana", Equals, uint(1)))
testSuccess(t, newIntCondition("banana", Equals, uint8(1)))
testSuccess(t, newIntCondition("banana", Equals, uint16(1)))
testSuccess(t, newIntCondition("banana", Equals, uint32(1)))
testSuccess(t, newIntCondition("banana", Equals, int(1)))
testSuccess(t, newIntCondition("banana", Equals, int8(1)))
testSuccess(t, newIntCondition("banana", Equals, int16(1)))
testSuccess(t, newIntCondition("banana", Equals, int32(1)))
testSuccess(t, newIntCondition("banana", Equals, int64(1)))
testSuccess(t, newIntCondition("banana", Equals, "1"))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1"))
testSuccess(t, newStringCondition("banana", SameAs, "coconut"))
testSuccess(t, newRegexCondition("banana", Matches, "coconut"))
testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"}))
testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut"))
}
func testCondError(t *testing.T, c Condition) {
t.Helper()
err := c.check()
if err == nil {
t.Error("should fail")
}
}
func TestConditionErrors(t *testing.T) {
t.Parallel()
// test invalid value types
testCondError(t, newBoolCondition("banana", Is, 1))
testCondError(t, newFloatCondition("banana", FloatEquals, true))
testCondError(t, newIntCondition("banana", Equals, true))
testCondError(t, newStringCondition("banana", SameAs, 1))
testCondError(t, newRegexCondition("banana", Matches, 1))
testCondError(t, newStringSliceCondition("banana", Matches, 1))
// test error presence
testCondError(t, newBoolCondition("banana", errorPresent, true))
testCondError(t, And(newBoolCondition("banana", errorPresent, true)))
testCondError(t, Or(newBoolCondition("banana", errorPresent, true)))
testCondError(t, newExistsCondition("banana", errorPresent))
testCondError(t, newFloatCondition("banana", errorPresent, 1.1))
testCondError(t, newIntCondition("banana", errorPresent, 1))
testCondError(t, newStringCondition("banana", errorPresent, "coconut"))
testCondError(t, newRegexCondition("banana", errorPresent, "coconut"))
}
func TestWhere(t *testing.T) {
t.Parallel()
c := Where("", 254, nil)
err := c.check()
if err == nil {
t.Error("should fail")
}
}

View File

@@ -0,0 +1,53 @@
package query
var (
operatorNames = map[string]uint8{
"==": Equals,
">": GreaterThan,
">=": GreaterThanOrEqual,
"<": LessThan,
"<=": LessThanOrEqual,
"f==": FloatEquals,
"f>": FloatGreaterThan,
"f>=": FloatGreaterThanOrEqual,
"f<": FloatLessThan,
"f<=": FloatLessThanOrEqual,
"sameas": SameAs,
"s==": SameAs,
"contains": Contains,
"co": Contains,
"startswith": StartsWith,
"sw": StartsWith,
"endswith": EndsWith,
"ew": EndsWith,
"in": In,
"matches": Matches,
"re": Matches,
"is": Is,
"exists": Exists,
"ex": Exists,
}
primaryNames = make(map[uint8]string)
)
func init() {
for opName, opID := range operatorNames {
name, ok := primaryNames[opID]
if ok {
if len(name) < len(opName) {
primaryNames[opID] = opName
}
} else {
primaryNames[opID] = opName
}
}
}
func getOpName(operator uint8) string {
name, ok := primaryNames[operator]
if ok {
return name
}
return "[unknown]"
}

View File

@@ -0,0 +1,11 @@
package query
import "testing"
func TestGetOpName(t *testing.T) {
t.Parallel()
if getOpName(254) != "[unknown]" {
t.Error("unexpected output")
}
}

View File

@@ -0,0 +1,350 @@
package query
import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
type snippet struct {
text string
globalPosition int
}
// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
//
//nolint:gocognit
func ParseQuery(query string) (*Query, error) {
snippets, err := extractSnippets(query)
if err != nil {
return nil, err
}
snippetsPos := 0
getSnippet := func() (*snippet, error) {
// order is important, as parseAndOr will always consume one additional snippet.
snippetsPos++
if snippetsPos > len(snippets) {
return nil, fmt.Errorf("unexpected end at position %d", len(query))
}
return snippets[snippetsPos-1], nil
}
remainingSnippets := func() int {
return len(snippets) - snippetsPos
}
// check for query word
queryWord, err := getSnippet()
if err != nil {
return nil, err
}
if queryWord.text != "query" {
return nil, errors.New("queries must start with \"query\"")
}
// get prefix
prefix, err := getSnippet()
if err != nil {
return nil, err
}
q := New(prefix.text)
for remainingSnippets() > 0 {
command, err := getSnippet()
if err != nil {
return nil, err
}
switch command.text {
case "where":
if q.where != nil {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
// parse conditions
condition, err := parseAndOr(getSnippet, remainingSnippets, true)
if err != nil {
return nil, err
}
// go one back, as parseAndOr had to check if its done
snippetsPos--
q.Where(condition)
case "orderby":
if q.orderBy != "" {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
orderBySnippet, err := getSnippet()
if err != nil {
return nil, err
}
q.OrderBy(orderBySnippet.text)
case "limit":
if q.limit != 0 {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
limitSnippet, err := getSnippet()
if err != nil {
return nil, err
}
limit, err := strconv.ParseUint(limitSnippet.text, 10, 31)
if err != nil {
return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition)
}
q.Limit(int(limit))
case "offset":
if q.offset != 0 {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
offsetSnippet, err := getSnippet()
if err != nil {
return nil, err
}
offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31)
if err != nil {
return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition)
}
q.Offset(int(offset))
default:
return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition)
}
}
return q.Check()
}
func extractSnippets(text string) (snippets []*snippet, err error) {
skip := false
start := -1
inParenthesis := false
var pos int
var char rune
for pos, char = range text {
// skip
if skip {
skip = false
continue
}
if char == '\\' {
skip = true
}
// wait for parenthesis to be overs
if inParenthesis {
if char == '"' {
snippets = append(snippets, &snippet{
text: prepToken(text[start+1 : pos]),
globalPosition: start + 1,
})
start = -1
inParenthesis = false
}
continue
}
// handle segments
switch char {
case '\t', '\n', '\r', ' ', '(', ')':
if start >= 0 {
snippets = append(snippets, &snippet{
text: prepToken(text[start:pos]),
globalPosition: start + 1,
})
start = -1
}
default:
if start == -1 {
start = pos
}
}
// handle special segment characters
switch char {
case '(', ')':
snippets = append(snippets, &snippet{
text: text[pos : pos+1],
globalPosition: pos + 1,
})
case '"':
if start < pos {
return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1)
}
inParenthesis = true
}
}
// add last
if start >= 0 {
snippets = append(snippets, &snippet{
text: prepToken(text[start : pos+1]),
globalPosition: start + 1,
})
}
return snippets, nil
}
//nolint:gocognit
func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) {
var (
isOr = false
typeSet = false
wrapInNot = false
expectingMore = true
conditions []Condition
)
for {
if !expectingMore && rootCondition && remainingSnippets() == 0 {
// advance snippetsPos by one, as it will be set back by 1
_, _ = getSnippet()
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
}
firstSnippet, err := getSnippet()
if err != nil {
return nil, err
}
if !expectingMore && rootCondition {
switch firstSnippet.text {
case "orderby", "limit", "offset":
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
}
}
switch firstSnippet.text {
case "(":
condition, err := parseAndOr(getSnippet, remainingSnippets, false)
if err != nil {
return nil, err
}
if wrapInNot {
conditions = append(conditions, Not(condition))
wrapInNot = false
} else {
conditions = append(conditions, condition)
}
expectingMore = true
case ")":
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
case "and":
if typeSet && isOr {
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
}
isOr = false
typeSet = true
expectingMore = true
case "or":
if typeSet && !isOr {
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
}
isOr = true
typeSet = true
expectingMore = true
case "not":
wrapInNot = true
expectingMore = true
default:
condition, err := parseCondition(firstSnippet, getSnippet)
if err != nil {
return nil, err
}
if wrapInNot {
conditions = append(conditions, Not(condition))
wrapInNot = false
} else {
conditions = append(conditions, condition)
}
expectingMore = false
}
}
}
func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) {
wrapInNot := false
// get operator name
opName, err := getSnippet()
if err != nil {
return nil, err
}
// negate?
if opName.text == "not" {
wrapInNot = true
opName, err = getSnippet()
if err != nil {
return nil, err
}
}
// get operator
operator, ok := operatorNames[opName.text]
if !ok {
return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition)
}
// don't need a value for "exists"
if operator == Exists {
if wrapInNot {
return Not(Where(firstSnippet.text, operator, nil)), nil
}
return Where(firstSnippet.text, operator, nil), nil
}
// get value
value, err := getSnippet()
if err != nil {
return nil, err
}
if wrapInNot {
return Not(Where(firstSnippet.text, operator, value.text)), nil
}
return Where(firstSnippet.text, operator, value.text), nil
}
var escapeReplacer = regexp.MustCompile(`\\([^\\])`)
// prepToken removes surrounding parenthesis and escape characters.
func prepToken(text string) string {
return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
}
// escapeString correctly escapes a snippet for printing.
func escapeString(token string) string {
// check if token contains characters that need to be escaped
if strings.ContainsAny(token, "()\"\\\t\r\n ") {
// put the token in parenthesis and only escape \ and "
return fmt.Sprintf("\"%s\"", strings.ReplaceAll(token, "\"", "\\\""))
}
return token
}

View File

@@ -0,0 +1,177 @@
package query
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
)
func TestExtractSnippets(t *testing.T) {
t.Parallel()
text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ `
result1 := []*snippet{
{text: "query", globalPosition: 1},
{text: "test:", globalPosition: 7},
{text: "where", globalPosition: 13},
{text: "(", globalPosition: 19},
{text: "bananas", globalPosition: 21},
{text: ">", globalPosition: 31},
{text: "100", globalPosition: 33},
{text: "and", globalPosition: 37},
{text: "monkeys.#", globalPosition: 41},
{text: "<=", globalPosition: 51},
{text: "12", globalPosition: 54},
{text: ")", globalPosition: 58},
{text: "or", globalPosition: 59},
{text: "(", globalPosition: 61},
{text: "coconuts", globalPosition: 62},
{text: "<", globalPosition: 71},
{text: "10", globalPosition: 73},
{text: "and", globalPosition: 76},
{text: "area", globalPosition: 82},
{text: ">", globalPosition: 87},
{text: "50", globalPosition: 89},
{text: ")", globalPosition: 91},
{text: "or", globalPosition: 93},
{text: "name", globalPosition: 96},
{text: "sameas", globalPosition: 101},
{text: "Julian", globalPosition: 108},
{text: "or", globalPosition: 115},
{text: "name", globalPosition: 118},
{text: "matches", globalPosition: 123},
{text: "^King ", globalPosition: 131},
}
snippets, err := extractSnippets(text1)
if err != nil {
t.Errorf("failed to extract snippets: %s", err)
}
if !reflect.DeepEqual(result1, snippets) {
t.Errorf("unexpected results:")
for _, el := range snippets {
t.Errorf("%+v", el)
}
}
// t.Error(spew.Sprintf("%v", treeElement))
}
func testParsing(t *testing.T, queryText string, expectedResult *Query) {
t.Helper()
_, err := expectedResult.Check()
if err != nil {
t.Errorf("failed to create query: %s", err)
return
}
q, err := ParseQuery(queryText)
if err != nil {
t.Errorf("failed to parse query: %s", err)
return
}
if queryText != q.Print() {
t.Errorf("string match failed: %s", q.Print())
return
}
if !reflect.DeepEqual(expectedResult, q) {
t.Error("deepqual match failed.")
t.Error("got:")
t.Error(spew.Sdump(q))
t.Error("expected:")
t.Error(spew.Sdump(expectedResult))
}
}
func TestParseQuery(t *testing.T) {
t.Parallel()
text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20`
result1 := New("test:").Where(Or(
And(
Where("bananas", GreaterThan, 100),
Where("monkeys.#", LessThanOrEqual, 12),
),
Not(And(
Where("coconuts", LessThan, 10),
Not(Where("area", GreaterThan, 50)),
)),
Where("name", SameAs, "Julian"),
Where("name", Matches, "^King "),
)).OrderBy("name").Limit(10).Offset(20)
testParsing(t, text1, result1)
testParsing(t, `query test: orderby name`, New("test:").OrderBy("name"))
testParsing(t, `query test: limit 10`, New("test:").Limit(10))
testParsing(t, `query test: offset 10`, New("test:").Offset(10))
testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban")))
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
// test all operators
testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1)))
testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1)))
testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1)))
testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1)))
testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1)))
testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1)))
testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1)))
testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1)))
testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1)))
testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1)))
testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana")))
testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana")))
testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana")))
testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana")))
testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"})))
testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana")))
testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true)))
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
// special
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
}
func testParseError(t *testing.T, queryText string, expectedErrorString string) {
t.Helper()
_, err := ParseQuery(queryText)
if err == nil {
t.Errorf("should fail to parse: %s", queryText)
return
}
if err.Error() != expectedErrorString {
t.Errorf("unexpected error for query: %s\nwanted: %s\n got: %s", queryText, expectedErrorString, err)
}
}
func TestParseErrors(t *testing.T) {
t.Parallel()
// syntax
testParseError(t, `query`, `unexpected end at position 5`)
testParseError(t, `query test: where`, `unexpected end at position 17`)
testParseError(t, `query test: where (`, `unexpected end at position 19`)
testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
testParseError(t, `query test: where not`, `unexpected end at position 21`)
testParseError(t, `query test: where banana`, `unexpected end at position 24`)
testParseError(t, `query test: where banana >`, `unexpected end at position 26`)
testParseError(t, `query test: where banana nope`, `unknown operator at position 26`)
testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`)
testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`)
testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`)
testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`)
testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`)
// testParseError(t, `query test: where banana exists and (`, ``)
// value parsing error
testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`)
testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`)
testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`)
testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`")
testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`)
}

View File

@@ -0,0 +1,170 @@
package query
import (
"fmt"
"strings"
"github.com/safing/portmaster/base/database/accessor"
"github.com/safing/portmaster/base/database/record"
)
// Example:
// q.New("core:/",
// q.Where("a", q.GreaterThan, 0),
// q.Where("b", q.Equals, 0),
// q.Or(
// q.Where("c", q.StartsWith, "x"),
// q.Where("d", q.Contains, "y")
// )
// )
// Query contains a compiled query.
type Query struct {
checked bool
dbName string
dbKeyPrefix string
where Condition
orderBy string
limit int
offset int
}
// New creates a new query with the supplied prefix.
func New(prefix string) *Query {
dbName, dbKeyPrefix := record.ParseKey(prefix)
return &Query{
dbName: dbName,
dbKeyPrefix: dbKeyPrefix,
}
}
// Where adds filtering.
func (q *Query) Where(condition Condition) *Query {
q.where = condition
return q
}
// Limit limits the number of returned results.
func (q *Query) Limit(limit int) *Query {
q.limit = limit
return q
}
// Offset sets the query offset.
func (q *Query) Offset(offset int) *Query {
q.offset = offset
return q
}
// OrderBy orders the results by the given key.
func (q *Query) OrderBy(key string) *Query {
q.orderBy = key
return q
}
// Check checks for errors in the query.
func (q *Query) Check() (*Query, error) {
if q.checked {
return q, nil
}
// check condition
if q.where != nil {
err := q.where.check()
if err != nil {
return nil, err
}
}
q.checked = true
return q, nil
}
// MustBeValid checks for errors in the query and panics if there is an error.
func (q *Query) MustBeValid() *Query {
_, err := q.Check()
if err != nil {
panic(err)
}
return q
}
// IsChecked returns whether they query was checked.
func (q *Query) IsChecked() bool {
return q.checked
}
// MatchesKey checks whether the query matches the supplied database key (key without database prefix).
func (q *Query) MatchesKey(dbKey string) bool {
return strings.HasPrefix(dbKey, q.dbKeyPrefix)
}
// MatchesRecord checks whether the query matches the supplied database record (value only).
func (q *Query) MatchesRecord(r record.Record) bool {
if q.where == nil {
return true
}
acc := r.GetAccessor(r)
if acc == nil {
return false
}
return q.where.complies(acc)
}
// MatchesAccessor checks whether the query matches the supplied accessor (value only).
func (q *Query) MatchesAccessor(acc accessor.Accessor) bool {
if q.where == nil {
return true
}
return q.where.complies(acc)
}
// Matches checks whether the query matches the supplied database record.
func (q *Query) Matches(r record.Record) bool {
if !q.MatchesKey(r.DatabaseKey()) {
return false
}
return q.MatchesRecord(r)
}
// Print returns the string representation of the query.
func (q *Query) Print() string {
var where string
if q.where != nil {
where = q.where.string()
if where != "" {
if strings.HasPrefix(where, "(") {
where = where[1 : len(where)-1]
}
where = fmt.Sprintf(" where %s", where)
}
}
var orderBy string
if q.orderBy != "" {
orderBy = fmt.Sprintf(" orderby %s", q.orderBy)
}
var limit string
if q.limit > 0 {
limit = fmt.Sprintf(" limit %d", q.limit)
}
var offset string
if q.offset > 0 {
offset = fmt.Sprintf(" offset %d", q.offset)
}
return fmt.Sprintf("query %s:%s%s%s%s%s", q.dbName, q.dbKeyPrefix, where, orderBy, limit, offset)
}
// DatabaseName returns the name of the database.
func (q *Query) DatabaseName() string {
return q.dbName
}
// DatabaseKeyPrefix returns the key prefix for the database.
func (q *Query) DatabaseKeyPrefix() string {
return q.dbKeyPrefix
}

View File

@@ -0,0 +1,113 @@
//nolint:unparam
package query
import (
"testing"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/structures/dsd"
)
// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
var testJSON = `{"age":100, "name":{"here":"B\\\"R"},
"noop":{"what is a wren?":"a bird"},
"happy":true,"immortal":false,
"items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
"arr":["1",2,"3",{"hello":"world"},"4",5],
"vals":[1,2,3,{"sadf":sdf"asdf"}],"name":{"first":"tom","last":null},
"created":"2014-05-16T08:28:06.989Z",
"loggy":{
"programmers": [
{
"firstName": "Brett",
"lastName": "McLaughlin",
"email": "aaaa",
"tag": "good"
},
{
"firstName": "Jason",
"lastName": "Hunter",
"email": "bbbb",
"tag": "bad"
},
{
"firstName": "Elliotte",
"lastName": "Harold",
"email": "cccc",
"tag":, "good"
},
{
"firstName": 1002.3,
"age": 101
}
]
},
"lastly":{"yay":"final"},
"temperature": 120.413
}`
func testQuery(t *testing.T, r record.Record, shouldMatch bool, condition Condition) {
t.Helper()
q := New("test:").Where(condition).MustBeValid()
// fmt.Printf("%s\n", q.Print())
matched := q.Matches(r)
switch {
case !matched && shouldMatch:
t.Errorf("should match: %s", q.Print())
case matched && !shouldMatch:
t.Errorf("should not match: %s", q.Print())
}
}
func TestQuery(t *testing.T) {
t.Parallel()
// if !gjson.Valid(testJSON) {
// t.Fatal("test json is invalid")
// }
r, err := record.NewWrapper("", nil, dsd.JSON, []byte(testJSON))
if err != nil {
t.Fatal(err)
}
testQuery(t, r, true, Where("age", Equals, 100))
testQuery(t, r, true, Where("age", GreaterThan, uint8(99)))
testQuery(t, r, true, Where("age", GreaterThanOrEqual, 99))
testQuery(t, r, true, Where("age", GreaterThanOrEqual, 100))
testQuery(t, r, true, Where("age", LessThan, 101))
testQuery(t, r, true, Where("age", LessThanOrEqual, "101"))
testQuery(t, r, true, Where("age", LessThanOrEqual, 100))
testQuery(t, r, true, Where("temperature", FloatEquals, 120.413))
testQuery(t, r, true, Where("temperature", FloatGreaterThan, 120))
testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120))
testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120.413))
testQuery(t, r, true, Where("temperature", FloatLessThan, 121))
testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "121"))
testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
testQuery(t, r, true, Where("lastly.yay", SameAs, "final"))
testQuery(t, r, true, Where("lastly.yay", Contains, "ina"))
testQuery(t, r, true, Where("lastly.yay", StartsWith, "fin"))
testQuery(t, r, true, Where("lastly.yay", EndsWith, "nal"))
testQuery(t, r, true, Where("lastly.yay", In, "draft,final"))
testQuery(t, r, true, Where("lastly.yay", In, "final,draft"))
testQuery(t, r, true, Where("happy", Is, true))
testQuery(t, r, true, Where("happy", Is, "true"))
testQuery(t, r, true, Where("happy", Is, "t"))
testQuery(t, r, true, Not(Where("happy", Is, "0")))
testQuery(t, r, true, And(
Where("happy", Is, "1"),
Not(Or(
Where("happy", Is, false),
Where("happy", Is, "f"),
)),
))
testQuery(t, r, true, Where("happy", Exists, nil))
testQuery(t, r, true, Where("created", Matches, "^2014-[0-9]{2}-[0-9]{2}T"))
}