diff --git a/netquery/query.go b/netquery/query.go index 929255ee..2cd2d9ef 100644 --- a/netquery/query.go +++ b/netquery/query.go @@ -59,6 +59,8 @@ type ( OrderBy OrderBys `json:"orderBy"` GroupBy []string `json:"groupBy"` + Pagination + selectedFields []string whitelistedFields []string paramMap map[string]interface{} @@ -74,6 +76,11 @@ type ( } OrderBys []OrderBy + + Pagination struct { + PageSize int `json:"pageSize"` + Page int `json:"page"` + } ) func (query *Query) UnmarshalJSON(blob []byte) error { @@ -156,6 +163,30 @@ func (query *Query) UnmarshalJSON(blob []byte) error { return nil } +// TODO(ppacher): right now we only support LIMIT and OFFSET for pagination but that +// has an issue that loading the same page twice might yield different results due to +// new records shifting the result slice. To overcome this, return a "PageToken" to the +// user that includes the time the initial query was created so paginated queries can +// ensure new records don't end up in the result set. +func (page *Pagination) toSQLLimitOffsetClause() string { + limit := page.PageSize + + // default and cap the limit to at most 100 items + // per page to avoid out-of-memory conditions when loading + // thousands of results at once. + if limit <= 0 || limit > 100 { + limit = 100 + } + + sql := fmt.Sprintf("LIMIT %d", limit) + + if page.Page > 0 { + sql += fmt.Sprintf(" OFFSET %d", page.Page*limit) + } + + return sql +} + func parseMatcher(raw json.RawMessage) (*Matcher, error) { var m Matcher if err := json.Unmarshal(raw, &m); err != nil { diff --git a/netquery/query_handler.go b/netquery/query_handler.go index 318e494d..672a343c 100644 --- a/netquery/query_handler.go +++ b/netquery/query_handler.go @@ -181,9 +181,10 @@ func (req *QueryRequestPayload) generateSQL(ctx context.Context, schema *orm.Tab if whereClause != "" { query += " WHERE " + whereClause } - query += " " + groupByClause + " " + orderByClause - return query, req.paramMap, nil + query += " " + groupByClause + " " + orderByClause + " " + req.Pagination.toSQLLimitOffsetClause() + + return strings.TrimSpace(query), req.paramMap, nil } func (req *QueryRequestPayload) prepareSelectedFields(ctx context.Context, schema *orm.TableSchema) error {