Restructure modules (#1572)
* Move portbase into monorepo * Add new simple module mgr * [WIP] Switch to new simple module mgr * Add StateMgr and more worker variants * [WIP] Switch more modules * [WIP] Switch more modules * [WIP] swtich more modules * [WIP] switch all SPN modules * [WIP] switch all service modules * [WIP] Convert all workers to the new module system * [WIP] add new task system to module manager * [WIP] Add second take for scheduling workers * [WIP] Add FIXME for bugs in new scheduler * [WIP] Add minor improvements to scheduler * [WIP] Add new worker scheduler * [WIP] Fix more bug related to new module system * [WIP] Fix start handing of the new module system * [WIP] Improve startup process * [WIP] Fix minor issues * [WIP] Fix missing subsystem in settings * [WIP] Initialize managers in constructor * [WIP] Move module event initialization to constrictors * [WIP] Fix setting for enabling and disabling the SPN module * [WIP] Move API registeration into module construction * [WIP] Update states mgr for all modules * [WIP] Add CmdLine operation support * Add state helper methods to module group and instance * Add notification and module status handling to status package * Fix starting issues * Remove pilot widget and update security lock to new status data * Remove debug logs * Improve http server shutdown * Add workaround for cleanly shutting down firewall+netquery * Improve logging * Add syncing states with notifications for new module system * Improve starting, stopping, shutdown; resolve FIXMEs/TODOs * [WIP] Fix most unit tests * Review new module system and fix minor issues * Push shutdown and restart events again via API * Set sleep mode via interface * Update example/template module * [WIP] Fix spn/cabin unit test * Remove deprecated UI elements * Make log output more similar for the logging transition phase * Switch spn hub and observer cmds to new module system * Fix log sources * Make worker mgr less error prone * Fix tests and minor issues * Fix observation hub * Improve shutdown and restart handling * Split up big connection.go source file * Move varint and dsd packages to structures repo * Improve expansion test * Fix linter warnings * Fix interception module on windows * Fix linter errors --------- Co-authored-by: Vladimir Stoilov <vladimir@safing.io>
This commit is contained in:
55
base/database/query/README.md
Normal file
55
base/database/query/README.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Query
|
||||
|
||||
## Control Flow
|
||||
|
||||
- Grouping with `(` and `)`
|
||||
- Chaining with `and` and `or`
|
||||
- _NO_ mixing! Be explicit and use grouping.
|
||||
- Negation with `not`
|
||||
- in front of expression for group: `not (...)`
|
||||
- inside expression for clause: `name not matches "^King "`
|
||||
|
||||
## Selectors
|
||||
|
||||
Supported by all feeders:
|
||||
- root level field: `field`
|
||||
- sub level field: `field.sub`
|
||||
- array/slice/map access: `map.0`
|
||||
- array/slice/map length: `map.#`
|
||||
|
||||
Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys.
|
||||
|
||||
## Operators
|
||||
|
||||
| Name | Textual | Req. Type | Internal Type | Compared with |
|
||||
|-------------------------|--------------------|-----------|---------------|---------------------------|
|
||||
| Equals | `==` | int | int64 | `==` |
|
||||
| GreaterThan | `>` | int | int64 | `>` |
|
||||
| GreaterThanOrEqual | `>=` | int | int64 | `>=` |
|
||||
| LessThan | `<` | int | int64 | `<` |
|
||||
| LessThanOrEqual | `<=` | int | int64 | `<=` |
|
||||
| FloatEquals | `f==` | float | float64 | `==` |
|
||||
| FloatGreaterThan | `f>` | float | float64 | `>` |
|
||||
| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` |
|
||||
| FloatLessThan | `f<` | float | float64 | `<` |
|
||||
| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` |
|
||||
| SameAs | `sameas`, `s==` | string | string | `==` |
|
||||
| Contains | `contains`, `co` | string | string | `strings.Contains()` |
|
||||
| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` |
|
||||
| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` |
|
||||
| In | `in` | string | string | for loop with `==` |
|
||||
| Matches | `matches`, `re` | string | string | `regexp.Regexp.Matches()` |
|
||||
| Is | `is` | bool* | bool | `==` |
|
||||
| Exists | `exists`, `ex` | any | n/a | n/a |
|
||||
|
||||
\*accepts strings: 1, t, T, true, True, TRUE, 0, f, F, false, False, FALSE
|
||||
|
||||
## Escaping
|
||||
|
||||
If you need to use a control character within a value (ie. not for controlling), escape it with `\`.
|
||||
It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible.
|
||||
|
||||
| Location | Characters to be escaped |
|
||||
|---|---|
|
||||
| Within parenthesis (`"`) | `"`, `\` |
|
||||
| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) |
|
||||
46
base/database/query/condition-and.go
Normal file
46
base/database/query/condition-and.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
// And combines multiple conditions with a logical _AND_ operator.
|
||||
func And(conditions ...Condition) Condition {
|
||||
return &andCond{
|
||||
conditions: conditions,
|
||||
}
|
||||
}
|
||||
|
||||
type andCond struct {
|
||||
conditions []Condition
|
||||
}
|
||||
|
||||
func (c *andCond) complies(acc accessor.Accessor) bool {
|
||||
for _, cond := range c.conditions {
|
||||
if !cond.complies(acc) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *andCond) check() (err error) {
|
||||
for _, cond := range c.conditions {
|
||||
err = cond.check()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *andCond) string() string {
|
||||
all := make([]string, 0, len(c.conditions))
|
||||
for _, cond := range c.conditions {
|
||||
all = append(all, cond.string())
|
||||
}
|
||||
return fmt.Sprintf("(%s)", strings.Join(all, " and "))
|
||||
}
|
||||
69
base/database/query/condition-bool.go
Normal file
69
base/database/query/condition-bool.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type boolCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
value bool
|
||||
}
|
||||
|
||||
func newBoolCondition(key string, operator uint8, value interface{}) *boolCondition {
|
||||
var parsedValue bool
|
||||
|
||||
switch v := value.(type) {
|
||||
case bool:
|
||||
parsedValue = v
|
||||
case string:
|
||||
var err error
|
||||
parsedValue, err = strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return &boolCondition{
|
||||
key: fmt.Sprintf("could not parse \"%s\" to bool: %s", v, err),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
default:
|
||||
return &boolCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for int64", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
|
||||
return &boolCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: parsedValue,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *boolCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetBool(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case Is:
|
||||
return comp == c.value
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *boolCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *boolCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value)
|
||||
}
|
||||
27
base/database/query/condition-error.go
Normal file
27
base/database/query/condition-error.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type errorCondition struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func newErrorCondition(err error) *errorCondition {
|
||||
return &errorCondition{
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *errorCondition) complies(acc accessor.Accessor) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *errorCondition) check() error {
|
||||
return c.err
|
||||
}
|
||||
|
||||
func (c *errorCondition) string() string {
|
||||
return "[ERROR]"
|
||||
}
|
||||
35
base/database/query/condition-exists.go
Normal file
35
base/database/query/condition-exists.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type existsCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
}
|
||||
|
||||
func newExistsCondition(key string, operator uint8) *existsCondition {
|
||||
return &existsCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *existsCondition) complies(acc accessor.Accessor) bool {
|
||||
return acc.Exists(c.key)
|
||||
}
|
||||
|
||||
func (c *existsCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *existsCondition) string() string {
|
||||
return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator))
|
||||
}
|
||||
97
base/database/query/condition-float.go
Normal file
97
base/database/query/condition-float.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type floatCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
value float64
|
||||
}
|
||||
|
||||
func newFloatCondition(key string, operator uint8, value interface{}) *floatCondition {
|
||||
var parsedValue float64
|
||||
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
parsedValue = float64(v)
|
||||
case int8:
|
||||
parsedValue = float64(v)
|
||||
case int16:
|
||||
parsedValue = float64(v)
|
||||
case int32:
|
||||
parsedValue = float64(v)
|
||||
case int64:
|
||||
parsedValue = float64(v)
|
||||
case uint:
|
||||
parsedValue = float64(v)
|
||||
case uint8:
|
||||
parsedValue = float64(v)
|
||||
case uint16:
|
||||
parsedValue = float64(v)
|
||||
case uint32:
|
||||
parsedValue = float64(v)
|
||||
case float32:
|
||||
parsedValue = float64(v)
|
||||
case float64:
|
||||
parsedValue = v
|
||||
case string:
|
||||
var err error
|
||||
parsedValue, err = strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
return &floatCondition{
|
||||
key: fmt.Sprintf("could not parse %s to float64: %s", v, err),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
default:
|
||||
return &floatCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for float64", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
|
||||
return &floatCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: parsedValue,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *floatCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetFloat(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case FloatEquals:
|
||||
return comp == c.value
|
||||
case FloatGreaterThan:
|
||||
return comp > c.value
|
||||
case FloatGreaterThanOrEqual:
|
||||
return comp >= c.value
|
||||
case FloatLessThan:
|
||||
return comp < c.value
|
||||
case FloatLessThanOrEqual:
|
||||
return comp <= c.value
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *floatCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *floatCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value)
|
||||
}
|
||||
93
base/database/query/condition-int.go
Normal file
93
base/database/query/condition-int.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type intCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
value int64
|
||||
}
|
||||
|
||||
func newIntCondition(key string, operator uint8, value interface{}) *intCondition {
|
||||
var parsedValue int64
|
||||
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
parsedValue = int64(v)
|
||||
case int8:
|
||||
parsedValue = int64(v)
|
||||
case int16:
|
||||
parsedValue = int64(v)
|
||||
case int32:
|
||||
parsedValue = int64(v)
|
||||
case int64:
|
||||
parsedValue = v
|
||||
case uint:
|
||||
parsedValue = int64(v)
|
||||
case uint8:
|
||||
parsedValue = int64(v)
|
||||
case uint16:
|
||||
parsedValue = int64(v)
|
||||
case uint32:
|
||||
parsedValue = int64(v)
|
||||
case string:
|
||||
var err error
|
||||
parsedValue, err = strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
return &intCondition{
|
||||
key: fmt.Sprintf("could not parse %s to int64: %s (hint: use \"sameas\" to compare strings)", v, err),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
default:
|
||||
return &intCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for int64", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
|
||||
return &intCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: parsedValue,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *intCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetInt(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case Equals:
|
||||
return comp == c.value
|
||||
case GreaterThan:
|
||||
return comp > c.value
|
||||
case GreaterThanOrEqual:
|
||||
return comp >= c.value
|
||||
case LessThan:
|
||||
return comp < c.value
|
||||
case LessThanOrEqual:
|
||||
return comp <= c.value
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *intCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *intCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value)
|
||||
}
|
||||
36
base/database/query/condition-not.go
Normal file
36
base/database/query/condition-not.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
// Not negates the supplied condition.
|
||||
func Not(c Condition) Condition {
|
||||
return ¬Cond{
|
||||
notC: c,
|
||||
}
|
||||
}
|
||||
|
||||
type notCond struct {
|
||||
notC Condition
|
||||
}
|
||||
|
||||
func (c *notCond) complies(acc accessor.Accessor) bool {
|
||||
return !c.notC.complies(acc)
|
||||
}
|
||||
|
||||
func (c *notCond) check() error {
|
||||
return c.notC.check()
|
||||
}
|
||||
|
||||
func (c *notCond) string() string {
|
||||
next := c.notC.string()
|
||||
if strings.HasPrefix(next, "(") {
|
||||
return fmt.Sprintf("not %s", c.notC.string())
|
||||
}
|
||||
splitted := strings.Split(next, " ")
|
||||
return strings.Join(append([]string{splitted[0], "not"}, splitted[1:]...), " ")
|
||||
}
|
||||
46
base/database/query/condition-or.go
Normal file
46
base/database/query/condition-or.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
// Or combines multiple conditions with a logical _OR_ operator.
|
||||
func Or(conditions ...Condition) Condition {
|
||||
return &orCond{
|
||||
conditions: conditions,
|
||||
}
|
||||
}
|
||||
|
||||
type orCond struct {
|
||||
conditions []Condition
|
||||
}
|
||||
|
||||
func (c *orCond) complies(acc accessor.Accessor) bool {
|
||||
for _, cond := range c.conditions {
|
||||
if cond.complies(acc) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *orCond) check() (err error) {
|
||||
for _, cond := range c.conditions {
|
||||
err = cond.check()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *orCond) string() string {
|
||||
all := make([]string, 0, len(c.conditions))
|
||||
for _, cond := range c.conditions {
|
||||
all = append(all, cond.string())
|
||||
}
|
||||
return fmt.Sprintf("(%s)", strings.Join(all, " or "))
|
||||
}
|
||||
63
base/database/query/condition-regex.go
Normal file
63
base/database/query/condition-regex.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type regexCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
regex *regexp.Regexp
|
||||
}
|
||||
|
||||
func newRegexCondition(key string, operator uint8, value interface{}) *regexCondition {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
r, err := regexp.Compile(v)
|
||||
if err != nil {
|
||||
return ®exCondition{
|
||||
key: fmt.Sprintf("could not compile regex \"%s\": %s", v, err),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
return ®exCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
regex: r,
|
||||
}
|
||||
default:
|
||||
return ®exCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for string", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *regexCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetString(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case Matches:
|
||||
return c.regex.MatchString(comp)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *regexCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *regexCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String()))
|
||||
}
|
||||
62
base/database/query/condition-string.go
Normal file
62
base/database/query/condition-string.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
type stringCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
value string
|
||||
}
|
||||
|
||||
func newStringCondition(key string, operator uint8, value interface{}) *stringCondition {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return &stringCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: v,
|
||||
}
|
||||
default:
|
||||
return &stringCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for string", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *stringCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetString(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case SameAs:
|
||||
return c.value == comp
|
||||
case Contains:
|
||||
return strings.Contains(comp, c.value)
|
||||
case StartsWith:
|
||||
return strings.HasPrefix(comp, c.value)
|
||||
case EndsWith:
|
||||
return strings.HasSuffix(comp, c.value)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *stringCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return errors.New(c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *stringCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value))
|
||||
}
|
||||
69
base/database/query/condition-stringslice.go
Normal file
69
base/database/query/condition-stringslice.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
"github.com/safing/portmaster/base/utils"
|
||||
)
|
||||
|
||||
type stringSliceCondition struct {
|
||||
key string
|
||||
operator uint8
|
||||
value []string
|
||||
}
|
||||
|
||||
func newStringSliceCondition(key string, operator uint8, value interface{}) *stringSliceCondition {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
parsedValue := strings.Split(v, ",")
|
||||
if len(parsedValue) < 2 {
|
||||
return &stringSliceCondition{
|
||||
key: v,
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
return &stringSliceCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: parsedValue,
|
||||
}
|
||||
case []string:
|
||||
return &stringSliceCondition{
|
||||
key: key,
|
||||
operator: operator,
|
||||
value: v,
|
||||
}
|
||||
default:
|
||||
return &stringSliceCondition{
|
||||
key: fmt.Sprintf("incompatible value %v for []string", value),
|
||||
operator: errorPresent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *stringSliceCondition) complies(acc accessor.Accessor) bool {
|
||||
comp, ok := acc.GetString(c.key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
switch c.operator {
|
||||
case In:
|
||||
return utils.StringInSlice(c.value, comp)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *stringSliceCondition) check() error {
|
||||
if c.operator == errorPresent {
|
||||
return fmt.Errorf("could not parse \"%s\" to []string", c.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *stringSliceCondition) string() string {
|
||||
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ",")))
|
||||
}
|
||||
71
base/database/query/condition.go
Normal file
71
base/database/query/condition.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
)
|
||||
|
||||
// Condition is an interface to provide a common api to all condition types.
|
||||
type Condition interface {
|
||||
complies(acc accessor.Accessor) bool
|
||||
check() error
|
||||
string() string
|
||||
}
|
||||
|
||||
// Operators.
|
||||
const (
|
||||
Equals uint8 = iota // int
|
||||
GreaterThan // int
|
||||
GreaterThanOrEqual // int
|
||||
LessThan // int
|
||||
LessThanOrEqual // int
|
||||
FloatEquals // float
|
||||
FloatGreaterThan // float
|
||||
FloatGreaterThanOrEqual // float
|
||||
FloatLessThan // float
|
||||
FloatLessThanOrEqual // float
|
||||
SameAs // string
|
||||
Contains // string
|
||||
StartsWith // string
|
||||
EndsWith // string
|
||||
In // stringSlice
|
||||
Matches // regex
|
||||
Is // bool: accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE
|
||||
Exists // any
|
||||
|
||||
errorPresent uint8 = 255
|
||||
)
|
||||
|
||||
// Where returns a condition to add to a query.
|
||||
func Where(key string, operator uint8, value interface{}) Condition {
|
||||
switch operator {
|
||||
case Equals,
|
||||
GreaterThan,
|
||||
GreaterThanOrEqual,
|
||||
LessThan,
|
||||
LessThanOrEqual:
|
||||
return newIntCondition(key, operator, value)
|
||||
case FloatEquals,
|
||||
FloatGreaterThan,
|
||||
FloatGreaterThanOrEqual,
|
||||
FloatLessThan,
|
||||
FloatLessThanOrEqual:
|
||||
return newFloatCondition(key, operator, value)
|
||||
case SameAs,
|
||||
Contains,
|
||||
StartsWith,
|
||||
EndsWith:
|
||||
return newStringCondition(key, operator, value)
|
||||
case In:
|
||||
return newStringSliceCondition(key, operator, value)
|
||||
case Matches:
|
||||
return newRegexCondition(key, operator, value)
|
||||
case Is:
|
||||
return newBoolCondition(key, operator, value)
|
||||
case Exists:
|
||||
return newExistsCondition(key, operator)
|
||||
default:
|
||||
return newErrorCondition(fmt.Errorf("no operator with ID %d", operator))
|
||||
}
|
||||
}
|
||||
86
base/database/query/condition_test.go
Normal file
86
base/database/query/condition_test.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package query
|
||||
|
||||
import "testing"
|
||||
|
||||
func testSuccess(t *testing.T, c Condition) {
|
||||
t.Helper()
|
||||
|
||||
err := c.check()
|
||||
if err != nil {
|
||||
t.Errorf("failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterfaces(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
testSuccess(t, newIntCondition("banana", Equals, uint(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, uint8(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, uint16(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, uint32(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, int(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, int8(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, int16(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, int32(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, int64(1)))
|
||||
testSuccess(t, newIntCondition("banana", Equals, "1"))
|
||||
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, int(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1)))
|
||||
testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1"))
|
||||
|
||||
testSuccess(t, newStringCondition("banana", SameAs, "coconut"))
|
||||
testSuccess(t, newRegexCondition("banana", Matches, "coconut"))
|
||||
testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"}))
|
||||
testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut"))
|
||||
}
|
||||
|
||||
func testCondError(t *testing.T, c Condition) {
|
||||
t.Helper()
|
||||
|
||||
err := c.check()
|
||||
if err == nil {
|
||||
t.Error("should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// test invalid value types
|
||||
testCondError(t, newBoolCondition("banana", Is, 1))
|
||||
testCondError(t, newFloatCondition("banana", FloatEquals, true))
|
||||
testCondError(t, newIntCondition("banana", Equals, true))
|
||||
testCondError(t, newStringCondition("banana", SameAs, 1))
|
||||
testCondError(t, newRegexCondition("banana", Matches, 1))
|
||||
testCondError(t, newStringSliceCondition("banana", Matches, 1))
|
||||
|
||||
// test error presence
|
||||
testCondError(t, newBoolCondition("banana", errorPresent, true))
|
||||
testCondError(t, And(newBoolCondition("banana", errorPresent, true)))
|
||||
testCondError(t, Or(newBoolCondition("banana", errorPresent, true)))
|
||||
testCondError(t, newExistsCondition("banana", errorPresent))
|
||||
testCondError(t, newFloatCondition("banana", errorPresent, 1.1))
|
||||
testCondError(t, newIntCondition("banana", errorPresent, 1))
|
||||
testCondError(t, newStringCondition("banana", errorPresent, "coconut"))
|
||||
testCondError(t, newRegexCondition("banana", errorPresent, "coconut"))
|
||||
}
|
||||
|
||||
func TestWhere(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := Where("", 254, nil)
|
||||
err := c.check()
|
||||
if err == nil {
|
||||
t.Error("should fail")
|
||||
}
|
||||
}
|
||||
53
base/database/query/operators.go
Normal file
53
base/database/query/operators.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package query
|
||||
|
||||
var (
|
||||
operatorNames = map[string]uint8{
|
||||
"==": Equals,
|
||||
">": GreaterThan,
|
||||
">=": GreaterThanOrEqual,
|
||||
"<": LessThan,
|
||||
"<=": LessThanOrEqual,
|
||||
"f==": FloatEquals,
|
||||
"f>": FloatGreaterThan,
|
||||
"f>=": FloatGreaterThanOrEqual,
|
||||
"f<": FloatLessThan,
|
||||
"f<=": FloatLessThanOrEqual,
|
||||
"sameas": SameAs,
|
||||
"s==": SameAs,
|
||||
"contains": Contains,
|
||||
"co": Contains,
|
||||
"startswith": StartsWith,
|
||||
"sw": StartsWith,
|
||||
"endswith": EndsWith,
|
||||
"ew": EndsWith,
|
||||
"in": In,
|
||||
"matches": Matches,
|
||||
"re": Matches,
|
||||
"is": Is,
|
||||
"exists": Exists,
|
||||
"ex": Exists,
|
||||
}
|
||||
|
||||
primaryNames = make(map[uint8]string)
|
||||
)
|
||||
|
||||
func init() {
|
||||
for opName, opID := range operatorNames {
|
||||
name, ok := primaryNames[opID]
|
||||
if ok {
|
||||
if len(name) < len(opName) {
|
||||
primaryNames[opID] = opName
|
||||
}
|
||||
} else {
|
||||
primaryNames[opID] = opName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getOpName(operator uint8) string {
|
||||
name, ok := primaryNames[operator]
|
||||
if ok {
|
||||
return name
|
||||
}
|
||||
return "[unknown]"
|
||||
}
|
||||
11
base/database/query/operators_test.go
Normal file
11
base/database/query/operators_test.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package query
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetOpName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if getOpName(254) != "[unknown]" {
|
||||
t.Error("unexpected output")
|
||||
}
|
||||
}
|
||||
350
base/database/query/parser.go
Normal file
350
base/database/query/parser.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type snippet struct {
|
||||
text string
|
||||
globalPosition int
|
||||
}
|
||||
|
||||
// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
|
||||
//
|
||||
//nolint:gocognit
|
||||
func ParseQuery(query string) (*Query, error) {
|
||||
snippets, err := extractSnippets(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snippetsPos := 0
|
||||
|
||||
getSnippet := func() (*snippet, error) {
|
||||
// order is important, as parseAndOr will always consume one additional snippet.
|
||||
snippetsPos++
|
||||
if snippetsPos > len(snippets) {
|
||||
return nil, fmt.Errorf("unexpected end at position %d", len(query))
|
||||
}
|
||||
return snippets[snippetsPos-1], nil
|
||||
}
|
||||
remainingSnippets := func() int {
|
||||
return len(snippets) - snippetsPos
|
||||
}
|
||||
|
||||
// check for query word
|
||||
queryWord, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if queryWord.text != "query" {
|
||||
return nil, errors.New("queries must start with \"query\"")
|
||||
}
|
||||
|
||||
// get prefix
|
||||
prefix, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := New(prefix.text)
|
||||
|
||||
for remainingSnippets() > 0 {
|
||||
command, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch command.text {
|
||||
case "where":
|
||||
if q.where != nil {
|
||||
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
|
||||
}
|
||||
|
||||
// parse conditions
|
||||
condition, err := parseAndOr(getSnippet, remainingSnippets, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// go one back, as parseAndOr had to check if its done
|
||||
snippetsPos--
|
||||
|
||||
q.Where(condition)
|
||||
case "orderby":
|
||||
if q.orderBy != "" {
|
||||
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
|
||||
}
|
||||
|
||||
orderBySnippet, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q.OrderBy(orderBySnippet.text)
|
||||
case "limit":
|
||||
if q.limit != 0 {
|
||||
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
|
||||
}
|
||||
|
||||
limitSnippet, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit, err := strconv.ParseUint(limitSnippet.text, 10, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition)
|
||||
}
|
||||
|
||||
q.Limit(int(limit))
|
||||
case "offset":
|
||||
if q.offset != 0 {
|
||||
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
|
||||
}
|
||||
|
||||
offsetSnippet, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition)
|
||||
}
|
||||
|
||||
q.Offset(int(offset))
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition)
|
||||
}
|
||||
}
|
||||
|
||||
return q.Check()
|
||||
}
|
||||
|
||||
func extractSnippets(text string) (snippets []*snippet, err error) {
|
||||
skip := false
|
||||
start := -1
|
||||
inParenthesis := false
|
||||
var pos int
|
||||
var char rune
|
||||
|
||||
for pos, char = range text {
|
||||
|
||||
// skip
|
||||
if skip {
|
||||
skip = false
|
||||
continue
|
||||
}
|
||||
if char == '\\' {
|
||||
skip = true
|
||||
}
|
||||
|
||||
// wait for parenthesis to be overs
|
||||
if inParenthesis {
|
||||
if char == '"' {
|
||||
snippets = append(snippets, &snippet{
|
||||
text: prepToken(text[start+1 : pos]),
|
||||
globalPosition: start + 1,
|
||||
})
|
||||
start = -1
|
||||
inParenthesis = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// handle segments
|
||||
switch char {
|
||||
case '\t', '\n', '\r', ' ', '(', ')':
|
||||
if start >= 0 {
|
||||
snippets = append(snippets, &snippet{
|
||||
text: prepToken(text[start:pos]),
|
||||
globalPosition: start + 1,
|
||||
})
|
||||
start = -1
|
||||
}
|
||||
default:
|
||||
if start == -1 {
|
||||
start = pos
|
||||
}
|
||||
}
|
||||
|
||||
// handle special segment characters
|
||||
switch char {
|
||||
case '(', ')':
|
||||
snippets = append(snippets, &snippet{
|
||||
text: text[pos : pos+1],
|
||||
globalPosition: pos + 1,
|
||||
})
|
||||
case '"':
|
||||
if start < pos {
|
||||
return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1)
|
||||
}
|
||||
inParenthesis = true
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// add last
|
||||
if start >= 0 {
|
||||
snippets = append(snippets, &snippet{
|
||||
text: prepToken(text[start : pos+1]),
|
||||
globalPosition: start + 1,
|
||||
})
|
||||
}
|
||||
|
||||
return snippets, nil
|
||||
}
|
||||
|
||||
//nolint:gocognit
|
||||
func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) {
|
||||
var (
|
||||
isOr = false
|
||||
typeSet = false
|
||||
wrapInNot = false
|
||||
expectingMore = true
|
||||
conditions []Condition
|
||||
)
|
||||
|
||||
for {
|
||||
if !expectingMore && rootCondition && remainingSnippets() == 0 {
|
||||
// advance snippetsPos by one, as it will be set back by 1
|
||||
_, _ = getSnippet()
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0], nil
|
||||
}
|
||||
if isOr {
|
||||
return Or(conditions...), nil
|
||||
}
|
||||
return And(conditions...), nil
|
||||
}
|
||||
|
||||
firstSnippet, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !expectingMore && rootCondition {
|
||||
switch firstSnippet.text {
|
||||
case "orderby", "limit", "offset":
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0], nil
|
||||
}
|
||||
if isOr {
|
||||
return Or(conditions...), nil
|
||||
}
|
||||
return And(conditions...), nil
|
||||
}
|
||||
}
|
||||
|
||||
switch firstSnippet.text {
|
||||
case "(":
|
||||
condition, err := parseAndOr(getSnippet, remainingSnippets, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wrapInNot {
|
||||
conditions = append(conditions, Not(condition))
|
||||
wrapInNot = false
|
||||
} else {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
expectingMore = true
|
||||
case ")":
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0], nil
|
||||
}
|
||||
if isOr {
|
||||
return Or(conditions...), nil
|
||||
}
|
||||
return And(conditions...), nil
|
||||
case "and":
|
||||
if typeSet && isOr {
|
||||
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
|
||||
}
|
||||
isOr = false
|
||||
typeSet = true
|
||||
expectingMore = true
|
||||
case "or":
|
||||
if typeSet && !isOr {
|
||||
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
|
||||
}
|
||||
isOr = true
|
||||
typeSet = true
|
||||
expectingMore = true
|
||||
case "not":
|
||||
wrapInNot = true
|
||||
expectingMore = true
|
||||
default:
|
||||
condition, err := parseCondition(firstSnippet, getSnippet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wrapInNot {
|
||||
conditions = append(conditions, Not(condition))
|
||||
wrapInNot = false
|
||||
} else {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
expectingMore = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) {
|
||||
wrapInNot := false
|
||||
|
||||
// get operator name
|
||||
opName, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// negate?
|
||||
if opName.text == "not" {
|
||||
wrapInNot = true
|
||||
opName, err = getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// get operator
|
||||
operator, ok := operatorNames[opName.text]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition)
|
||||
}
|
||||
|
||||
// don't need a value for "exists"
|
||||
if operator == Exists {
|
||||
if wrapInNot {
|
||||
return Not(Where(firstSnippet.text, operator, nil)), nil
|
||||
}
|
||||
return Where(firstSnippet.text, operator, nil), nil
|
||||
}
|
||||
|
||||
// get value
|
||||
value, err := getSnippet()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wrapInNot {
|
||||
return Not(Where(firstSnippet.text, operator, value.text)), nil
|
||||
}
|
||||
return Where(firstSnippet.text, operator, value.text), nil
|
||||
}
|
||||
|
||||
var escapeReplacer = regexp.MustCompile(`\\([^\\])`)
|
||||
|
||||
// prepToken removes surrounding parenthesis and escape characters.
|
||||
func prepToken(text string) string {
|
||||
return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
|
||||
}
|
||||
|
||||
// escapeString correctly escapes a snippet for printing.
|
||||
func escapeString(token string) string {
|
||||
// check if token contains characters that need to be escaped
|
||||
if strings.ContainsAny(token, "()\"\\\t\r\n ") {
|
||||
// put the token in parenthesis and only escape \ and "
|
||||
return fmt.Sprintf("\"%s\"", strings.ReplaceAll(token, "\"", "\\\""))
|
||||
}
|
||||
return token
|
||||
}
|
||||
177
base/database/query/parser_test.go
Normal file
177
base/database/query/parser_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
)
|
||||
|
||||
func TestExtractSnippets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ `
|
||||
result1 := []*snippet{
|
||||
{text: "query", globalPosition: 1},
|
||||
{text: "test:", globalPosition: 7},
|
||||
{text: "where", globalPosition: 13},
|
||||
{text: "(", globalPosition: 19},
|
||||
{text: "bananas", globalPosition: 21},
|
||||
{text: ">", globalPosition: 31},
|
||||
{text: "100", globalPosition: 33},
|
||||
{text: "and", globalPosition: 37},
|
||||
{text: "monkeys.#", globalPosition: 41},
|
||||
{text: "<=", globalPosition: 51},
|
||||
{text: "12", globalPosition: 54},
|
||||
{text: ")", globalPosition: 58},
|
||||
{text: "or", globalPosition: 59},
|
||||
{text: "(", globalPosition: 61},
|
||||
{text: "coconuts", globalPosition: 62},
|
||||
{text: "<", globalPosition: 71},
|
||||
{text: "10", globalPosition: 73},
|
||||
{text: "and", globalPosition: 76},
|
||||
{text: "area", globalPosition: 82},
|
||||
{text: ">", globalPosition: 87},
|
||||
{text: "50", globalPosition: 89},
|
||||
{text: ")", globalPosition: 91},
|
||||
{text: "or", globalPosition: 93},
|
||||
{text: "name", globalPosition: 96},
|
||||
{text: "sameas", globalPosition: 101},
|
||||
{text: "Julian", globalPosition: 108},
|
||||
{text: "or", globalPosition: 115},
|
||||
{text: "name", globalPosition: 118},
|
||||
{text: "matches", globalPosition: 123},
|
||||
{text: "^King ", globalPosition: 131},
|
||||
}
|
||||
|
||||
snippets, err := extractSnippets(text1)
|
||||
if err != nil {
|
||||
t.Errorf("failed to extract snippets: %s", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(result1, snippets) {
|
||||
t.Errorf("unexpected results:")
|
||||
for _, el := range snippets {
|
||||
t.Errorf("%+v", el)
|
||||
}
|
||||
}
|
||||
|
||||
// t.Error(spew.Sprintf("%v", treeElement))
|
||||
}
|
||||
|
||||
func testParsing(t *testing.T, queryText string, expectedResult *Query) {
|
||||
t.Helper()
|
||||
|
||||
_, err := expectedResult.Check()
|
||||
if err != nil {
|
||||
t.Errorf("failed to create query: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
q, err := ParseQuery(queryText)
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse query: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if queryText != q.Print() {
|
||||
t.Errorf("string match failed: %s", q.Print())
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(expectedResult, q) {
|
||||
t.Error("deepqual match failed.")
|
||||
t.Error("got:")
|
||||
t.Error(spew.Sdump(q))
|
||||
t.Error("expected:")
|
||||
t.Error(spew.Sdump(expectedResult))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseQuery(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20`
|
||||
result1 := New("test:").Where(Or(
|
||||
And(
|
||||
Where("bananas", GreaterThan, 100),
|
||||
Where("monkeys.#", LessThanOrEqual, 12),
|
||||
),
|
||||
Not(And(
|
||||
Where("coconuts", LessThan, 10),
|
||||
Not(Where("area", GreaterThan, 50)),
|
||||
)),
|
||||
Where("name", SameAs, "Julian"),
|
||||
Where("name", Matches, "^King "),
|
||||
)).OrderBy("name").Limit(10).Offset(20)
|
||||
testParsing(t, text1, result1)
|
||||
|
||||
testParsing(t, `query test: orderby name`, New("test:").OrderBy("name"))
|
||||
testParsing(t, `query test: limit 10`, New("test:").Limit(10))
|
||||
testParsing(t, `query test: offset 10`, New("test:").Offset(10))
|
||||
testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban")))
|
||||
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
|
||||
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
|
||||
|
||||
// test all operators
|
||||
testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1)))
|
||||
testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1)))
|
||||
testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1)))
|
||||
testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1)))
|
||||
testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1)))
|
||||
testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1)))
|
||||
testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1)))
|
||||
testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1)))
|
||||
testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1)))
|
||||
testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1)))
|
||||
testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana")))
|
||||
testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana")))
|
||||
testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana")))
|
||||
testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana")))
|
||||
testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"})))
|
||||
testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana")))
|
||||
testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true)))
|
||||
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
|
||||
|
||||
// special
|
||||
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
|
||||
}
|
||||
|
||||
func testParseError(t *testing.T, queryText string, expectedErrorString string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := ParseQuery(queryText)
|
||||
if err == nil {
|
||||
t.Errorf("should fail to parse: %s", queryText)
|
||||
return
|
||||
}
|
||||
if err.Error() != expectedErrorString {
|
||||
t.Errorf("unexpected error for query: %s\nwanted: %s\n got: %s", queryText, expectedErrorString, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// syntax
|
||||
testParseError(t, `query`, `unexpected end at position 5`)
|
||||
testParseError(t, `query test: where`, `unexpected end at position 17`)
|
||||
testParseError(t, `query test: where (`, `unexpected end at position 19`)
|
||||
testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
|
||||
testParseError(t, `query test: where not`, `unexpected end at position 21`)
|
||||
testParseError(t, `query test: where banana`, `unexpected end at position 24`)
|
||||
testParseError(t, `query test: where banana >`, `unexpected end at position 26`)
|
||||
testParseError(t, `query test: where banana nope`, `unknown operator at position 26`)
|
||||
testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`)
|
||||
testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`)
|
||||
testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`)
|
||||
testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`)
|
||||
testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`)
|
||||
// testParseError(t, `query test: where banana exists and (`, ``)
|
||||
|
||||
// value parsing error
|
||||
testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`)
|
||||
testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`)
|
||||
testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`)
|
||||
testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`")
|
||||
testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`)
|
||||
}
|
||||
170
base/database/query/query.go
Normal file
170
base/database/query/query.go
Normal file
@@ -0,0 +1,170 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/safing/portmaster/base/database/accessor"
|
||||
"github.com/safing/portmaster/base/database/record"
|
||||
)
|
||||
|
||||
// Example:
|
||||
// q.New("core:/",
|
||||
// q.Where("a", q.GreaterThan, 0),
|
||||
// q.Where("b", q.Equals, 0),
|
||||
// q.Or(
|
||||
// q.Where("c", q.StartsWith, "x"),
|
||||
// q.Where("d", q.Contains, "y")
|
||||
// )
|
||||
// )
|
||||
|
||||
// Query contains a compiled query.
|
||||
type Query struct {
|
||||
checked bool
|
||||
dbName string
|
||||
dbKeyPrefix string
|
||||
where Condition
|
||||
orderBy string
|
||||
limit int
|
||||
offset int
|
||||
}
|
||||
|
||||
// New creates a new query with the supplied prefix.
|
||||
func New(prefix string) *Query {
|
||||
dbName, dbKeyPrefix := record.ParseKey(prefix)
|
||||
return &Query{
|
||||
dbName: dbName,
|
||||
dbKeyPrefix: dbKeyPrefix,
|
||||
}
|
||||
}
|
||||
|
||||
// Where adds filtering.
|
||||
func (q *Query) Where(condition Condition) *Query {
|
||||
q.where = condition
|
||||
return q
|
||||
}
|
||||
|
||||
// Limit limits the number of returned results.
|
||||
func (q *Query) Limit(limit int) *Query {
|
||||
q.limit = limit
|
||||
return q
|
||||
}
|
||||
|
||||
// Offset sets the query offset.
|
||||
func (q *Query) Offset(offset int) *Query {
|
||||
q.offset = offset
|
||||
return q
|
||||
}
|
||||
|
||||
// OrderBy orders the results by the given key.
|
||||
func (q *Query) OrderBy(key string) *Query {
|
||||
q.orderBy = key
|
||||
return q
|
||||
}
|
||||
|
||||
// Check checks for errors in the query.
|
||||
func (q *Query) Check() (*Query, error) {
|
||||
if q.checked {
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// check condition
|
||||
if q.where != nil {
|
||||
err := q.where.check()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
q.checked = true
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// MustBeValid checks for errors in the query and panics if there is an error.
|
||||
func (q *Query) MustBeValid() *Query {
|
||||
_, err := q.Check()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// IsChecked returns whether they query was checked.
|
||||
func (q *Query) IsChecked() bool {
|
||||
return q.checked
|
||||
}
|
||||
|
||||
// MatchesKey checks whether the query matches the supplied database key (key without database prefix).
|
||||
func (q *Query) MatchesKey(dbKey string) bool {
|
||||
return strings.HasPrefix(dbKey, q.dbKeyPrefix)
|
||||
}
|
||||
|
||||
// MatchesRecord checks whether the query matches the supplied database record (value only).
|
||||
func (q *Query) MatchesRecord(r record.Record) bool {
|
||||
if q.where == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
acc := r.GetAccessor(r)
|
||||
if acc == nil {
|
||||
return false
|
||||
}
|
||||
return q.where.complies(acc)
|
||||
}
|
||||
|
||||
// MatchesAccessor checks whether the query matches the supplied accessor (value only).
|
||||
func (q *Query) MatchesAccessor(acc accessor.Accessor) bool {
|
||||
if q.where == nil {
|
||||
return true
|
||||
}
|
||||
return q.where.complies(acc)
|
||||
}
|
||||
|
||||
// Matches checks whether the query matches the supplied database record.
|
||||
func (q *Query) Matches(r record.Record) bool {
|
||||
if !q.MatchesKey(r.DatabaseKey()) {
|
||||
return false
|
||||
}
|
||||
return q.MatchesRecord(r)
|
||||
}
|
||||
|
||||
// Print returns the string representation of the query.
|
||||
func (q *Query) Print() string {
|
||||
var where string
|
||||
if q.where != nil {
|
||||
where = q.where.string()
|
||||
if where != "" {
|
||||
if strings.HasPrefix(where, "(") {
|
||||
where = where[1 : len(where)-1]
|
||||
}
|
||||
where = fmt.Sprintf(" where %s", where)
|
||||
}
|
||||
}
|
||||
|
||||
var orderBy string
|
||||
if q.orderBy != "" {
|
||||
orderBy = fmt.Sprintf(" orderby %s", q.orderBy)
|
||||
}
|
||||
|
||||
var limit string
|
||||
if q.limit > 0 {
|
||||
limit = fmt.Sprintf(" limit %d", q.limit)
|
||||
}
|
||||
|
||||
var offset string
|
||||
if q.offset > 0 {
|
||||
offset = fmt.Sprintf(" offset %d", q.offset)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("query %s:%s%s%s%s%s", q.dbName, q.dbKeyPrefix, where, orderBy, limit, offset)
|
||||
}
|
||||
|
||||
// DatabaseName returns the name of the database.
|
||||
func (q *Query) DatabaseName() string {
|
||||
return q.dbName
|
||||
}
|
||||
|
||||
// DatabaseKeyPrefix returns the key prefix for the database.
|
||||
func (q *Query) DatabaseKeyPrefix() string {
|
||||
return q.dbKeyPrefix
|
||||
}
|
||||
113
base/database/query/query_test.go
Normal file
113
base/database/query/query_test.go
Normal file
@@ -0,0 +1,113 @@
|
||||
//nolint:unparam
|
||||
package query
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/safing/portmaster/base/database/record"
|
||||
"github.com/safing/structures/dsd"
|
||||
)
|
||||
|
||||
// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
|
||||
var testJSON = `{"age":100, "name":{"here":"B\\\"R"},
|
||||
"noop":{"what is a wren?":"a bird"},
|
||||
"happy":true,"immortal":false,
|
||||
"items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
|
||||
"arr":["1",2,"3",{"hello":"world"},"4",5],
|
||||
"vals":[1,2,3,{"sadf":sdf"asdf"}],"name":{"first":"tom","last":null},
|
||||
"created":"2014-05-16T08:28:06.989Z",
|
||||
"loggy":{
|
||||
"programmers": [
|
||||
{
|
||||
"firstName": "Brett",
|
||||
"lastName": "McLaughlin",
|
||||
"email": "aaaa",
|
||||
"tag": "good"
|
||||
},
|
||||
{
|
||||
"firstName": "Jason",
|
||||
"lastName": "Hunter",
|
||||
"email": "bbbb",
|
||||
"tag": "bad"
|
||||
},
|
||||
{
|
||||
"firstName": "Elliotte",
|
||||
"lastName": "Harold",
|
||||
"email": "cccc",
|
||||
"tag":, "good"
|
||||
},
|
||||
{
|
||||
"firstName": 1002.3,
|
||||
"age": 101
|
||||
}
|
||||
]
|
||||
},
|
||||
"lastly":{"yay":"final"},
|
||||
"temperature": 120.413
|
||||
}`
|
||||
|
||||
func testQuery(t *testing.T, r record.Record, shouldMatch bool, condition Condition) {
|
||||
t.Helper()
|
||||
|
||||
q := New("test:").Where(condition).MustBeValid()
|
||||
// fmt.Printf("%s\n", q.Print())
|
||||
|
||||
matched := q.Matches(r)
|
||||
switch {
|
||||
case !matched && shouldMatch:
|
||||
t.Errorf("should match: %s", q.Print())
|
||||
case matched && !shouldMatch:
|
||||
t.Errorf("should not match: %s", q.Print())
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuery(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// if !gjson.Valid(testJSON) {
|
||||
// t.Fatal("test json is invalid")
|
||||
// }
|
||||
r, err := record.NewWrapper("", nil, dsd.JSON, []byte(testJSON))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testQuery(t, r, true, Where("age", Equals, 100))
|
||||
testQuery(t, r, true, Where("age", GreaterThan, uint8(99)))
|
||||
testQuery(t, r, true, Where("age", GreaterThanOrEqual, 99))
|
||||
testQuery(t, r, true, Where("age", GreaterThanOrEqual, 100))
|
||||
testQuery(t, r, true, Where("age", LessThan, 101))
|
||||
testQuery(t, r, true, Where("age", LessThanOrEqual, "101"))
|
||||
testQuery(t, r, true, Where("age", LessThanOrEqual, 100))
|
||||
|
||||
testQuery(t, r, true, Where("temperature", FloatEquals, 120.413))
|
||||
testQuery(t, r, true, Where("temperature", FloatGreaterThan, 120))
|
||||
testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120))
|
||||
testQuery(t, r, true, Where("temperature", FloatGreaterThanOrEqual, 120.413))
|
||||
testQuery(t, r, true, Where("temperature", FloatLessThan, 121))
|
||||
testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "121"))
|
||||
testQuery(t, r, true, Where("temperature", FloatLessThanOrEqual, "120.413"))
|
||||
|
||||
testQuery(t, r, true, Where("lastly.yay", SameAs, "final"))
|
||||
testQuery(t, r, true, Where("lastly.yay", Contains, "ina"))
|
||||
testQuery(t, r, true, Where("lastly.yay", StartsWith, "fin"))
|
||||
testQuery(t, r, true, Where("lastly.yay", EndsWith, "nal"))
|
||||
testQuery(t, r, true, Where("lastly.yay", In, "draft,final"))
|
||||
testQuery(t, r, true, Where("lastly.yay", In, "final,draft"))
|
||||
|
||||
testQuery(t, r, true, Where("happy", Is, true))
|
||||
testQuery(t, r, true, Where("happy", Is, "true"))
|
||||
testQuery(t, r, true, Where("happy", Is, "t"))
|
||||
testQuery(t, r, true, Not(Where("happy", Is, "0")))
|
||||
testQuery(t, r, true, And(
|
||||
Where("happy", Is, "1"),
|
||||
Not(Or(
|
||||
Where("happy", Is, false),
|
||||
Where("happy", Is, "f"),
|
||||
)),
|
||||
))
|
||||
|
||||
testQuery(t, r, true, Where("happy", Exists, nil))
|
||||
|
||||
testQuery(t, r, true, Where("created", Matches, "^2014-[0-9]{2}-[0-9]{2}T"))
|
||||
}
|
||||
Reference in New Issue
Block a user