Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,286 @@
|
||||
// Package planner compiles a query string (either syntax) into ir.Plan.
|
||||
// This is the single entry point querylang exposes to callers (the /query
|
||||
// HTTP handler) -- see Compile.
|
||||
package planner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ast"
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
"github.com/sentry/sentry/api/internal/querylang/parser"
|
||||
)
|
||||
|
||||
// Language selects which syntax a query is written in.
|
||||
type Language string
|
||||
|
||||
const (
|
||||
Auto Language = "" // detect from the query text (default)
|
||||
SQL Language = "sql"
|
||||
SPL Language = "spl" // the pipe syntax; named to match the query-language-reference doc
|
||||
)
|
||||
|
||||
const defaultLimit = 100
|
||||
|
||||
// Compile turns a query string into a Plan. language overrides
|
||||
// auto-detection; pass Auto to use the SELECT-prefix heuristic (see
|
||||
// /docs/query-language-design.md's "Detection" section) -- this exists
|
||||
// for the rare case a pipe query legitimately starts with the literal
|
||||
// word "select" as a bare search term.
|
||||
func Compile(query string, language Language, now time.Time) (*ir.Plan, error) {
|
||||
isSQL := language == SQL
|
||||
if language == Auto {
|
||||
isSQL = looksLikeSQL(query)
|
||||
}
|
||||
|
||||
if isSQL {
|
||||
if err := validateSelectOnly(query); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ir.Plan{RawSQL: strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))}, nil
|
||||
}
|
||||
|
||||
q, err := parser.Parse(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compileQuery(q, now)
|
||||
}
|
||||
|
||||
func looksLikeSQL(query string) bool {
|
||||
trimmed := strings.TrimSpace(query)
|
||||
if trimmed == "" {
|
||||
return false
|
||||
}
|
||||
fields := strings.Fields(trimmed)
|
||||
return len(fields) > 0 && strings.EqualFold(fields[0], "SELECT")
|
||||
}
|
||||
|
||||
func compileQuery(q *ast.Query, now time.Time) (*ir.Plan, error) {
|
||||
plan := &ir.Plan{}
|
||||
|
||||
textParts, err := compileBoolExpr(q.Base, plan, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, stage := range q.Pipes {
|
||||
switch s := stage.(type) {
|
||||
case ast.WhereStage:
|
||||
parts, err := compileBoolExpr(s.Expr, plan, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
textParts = append(textParts, parts...)
|
||||
case ast.StatsStage:
|
||||
agg, err := compileStats(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan.Aggregation = agg
|
||||
case ast.SortStage:
|
||||
for _, f := range s.Fields {
|
||||
plan.Sort = append(plan.Sort, ir.SortField{Field: f.Field, Desc: f.Desc})
|
||||
}
|
||||
case ast.FieldsStage:
|
||||
plan.Fields = append(plan.Fields, s.Fields...)
|
||||
case ast.HeadStage:
|
||||
n := defaultLimit
|
||||
if s.HasN {
|
||||
n = s.N
|
||||
}
|
||||
plan.Limit = &ir.Limit{N: n, Tail: false}
|
||||
case ast.TailStage:
|
||||
n := defaultLimit
|
||||
if s.HasN {
|
||||
n = s.N
|
||||
}
|
||||
plan.Limit = &ir.Limit{N: n, Tail: true}
|
||||
default:
|
||||
return nil, fmt.Errorf("internal error: unhandled pipe stage %T", stage)
|
||||
}
|
||||
}
|
||||
|
||||
if len(textParts) > 0 {
|
||||
plan.TextSearch = []ir.TextPredicate{{Query: strings.Join(textParts, " ")}}
|
||||
}
|
||||
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// textPart is one piece of a combined Tantivy query string, tagged with
|
||||
// the conjunction that precedes it (empty for the first piece).
|
||||
type textPart struct {
|
||||
conj string // "", "and", "or"
|
||||
query string
|
||||
}
|
||||
|
||||
// compileBoolExpr walks one bool_expr (the base search, or a `where`
|
||||
// stage's expression), populating plan.Filters and plan.TimeRange
|
||||
// directly, and returning free-text pieces for the caller to fold into
|
||||
// the combined Tantivy query string.
|
||||
//
|
||||
// Scope decision: "or" is only supported between free-text terms, which
|
||||
// Tantivy's own query parser handles natively once composed into one
|
||||
// string. "or" between structured comparisons/time-bounds is rejected
|
||||
// with a clear error rather than silently compiled as "and" -- see
|
||||
// /docs/query-language-reference.md's limitations section. This keeps
|
||||
// the executor's generated SQL a flat AND-only WHERE clause, which is
|
||||
// most of what real queries need; full boolean-tree support for
|
||||
// structured filters is future work if usage shows it's needed.
|
||||
func compileBoolExpr(expr ast.BoolExpr, plan *ir.Plan, now time.Time) ([]string, error) {
|
||||
var textParts []string
|
||||
|
||||
for i, term := range expr.Terms {
|
||||
conj := ""
|
||||
if i > 0 {
|
||||
conj = expr.Conjs[i-1]
|
||||
}
|
||||
|
||||
switch t := term.(type) {
|
||||
case ast.Comparison:
|
||||
if conj == "or" {
|
||||
return nil, fmt.Errorf("query error: \"or\" is not supported between structured filters (%q) in Phase 2 -- only between free-text search terms", t.Field)
|
||||
}
|
||||
plan.Filters = append(plan.Filters, ir.FilterPredicate{Field: t.Field, Op: t.Op, Value: t.Value})
|
||||
case ast.TimeBound:
|
||||
if conj == "or" {
|
||||
return nil, fmt.Errorf("query error: \"or\" is not supported on time bounds (%s) in Phase 2", t.Kind)
|
||||
}
|
||||
if err := applyTimeBound(plan, t, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case ast.FreeText:
|
||||
q := t.Query
|
||||
if strings.ContainsAny(q, " \t") {
|
||||
q = `"` + strings.ReplaceAll(q, `"`, `\"`) + `"`
|
||||
}
|
||||
if conj == "or" {
|
||||
textParts = append(textParts, "OR", q)
|
||||
} else if len(textParts) > 0 {
|
||||
textParts = append(textParts, "AND", q)
|
||||
} else {
|
||||
textParts = append(textParts, q)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("internal error: unhandled term %T", term)
|
||||
}
|
||||
}
|
||||
|
||||
return textParts, nil
|
||||
}
|
||||
|
||||
func applyTimeBound(plan *ir.Plan, t ast.TimeBound, now time.Time) error {
|
||||
when, err := resolveTimeExpr(t.Expr, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.TimeRange == nil {
|
||||
plan.TimeRange = &ir.TimeRange{}
|
||||
}
|
||||
switch t.Kind {
|
||||
case "earliest":
|
||||
plan.TimeRange.From = when
|
||||
case "latest":
|
||||
plan.TimeRange.To = when
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveTimeExpr(e ast.TimeExpr, now time.Time) (time.Time, error) {
|
||||
if !e.IsRelative {
|
||||
t, err := time.Parse(time.RFC3339, e.Absolute)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("query error: invalid absolute timestamp %q, want RFC3339 (e.g. 2026-08-14T00:00:00Z): %w", e.Absolute, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
switch e.RelativeUnit {
|
||||
case "s":
|
||||
d = time.Duration(e.RelativeN) * time.Second
|
||||
case "m":
|
||||
d = time.Duration(e.RelativeN) * time.Minute
|
||||
case "h":
|
||||
d = time.Duration(e.RelativeN) * time.Hour
|
||||
case "d":
|
||||
d = time.Duration(e.RelativeN) * 24 * time.Hour
|
||||
case "w":
|
||||
d = time.Duration(e.RelativeN) * 7 * 24 * time.Hour
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("internal error: unknown time unit %q", e.RelativeUnit)
|
||||
}
|
||||
if e.RelativeSign < 0 {
|
||||
d = -d
|
||||
}
|
||||
return now.Add(d), nil
|
||||
}
|
||||
|
||||
func compileStats(s ast.StatsStage) (*ir.Aggregation, error) {
|
||||
agg := &ir.Aggregation{GroupBy: s.By}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range s.Aggs {
|
||||
if a.Func != "count" && a.Field == "" {
|
||||
return nil, fmt.Errorf("query error: %s() requires a field, e.g. %s(latency_ms)", a.Func, a.Func)
|
||||
}
|
||||
alias := a.Alias
|
||||
if alias == "" {
|
||||
alias = defaultAggAlias(a)
|
||||
}
|
||||
if seen[alias] && a.Alias == "" {
|
||||
// Two unnamed aggs of the same shape would otherwise collide
|
||||
// (e.g. `stats sum(a), sum(b)` both defaulting to "sum") --
|
||||
// disambiguate by field name.
|
||||
alias = alias + "_" + a.Field
|
||||
}
|
||||
seen[alias] = true
|
||||
agg.Funcs = append(agg.Funcs, ir.AggFunc{Func: a.Func, Field: a.Field, Alias: alias})
|
||||
}
|
||||
return agg, nil
|
||||
}
|
||||
|
||||
func defaultAggAlias(a ast.AggCall) string {
|
||||
if a.Func == "count" {
|
||||
return "count"
|
||||
}
|
||||
return a.Func
|
||||
}
|
||||
|
||||
// --- SQL escape hatch validation, ported from the Phase 0/1
|
||||
// api/internal/queryapi/validate.go guard it replaces (see task 4) ---
|
||||
|
||||
// disallowedKeyword is defense-in-depth on top of the SELECT-only gate:
|
||||
// it catches mutating/administrative statements appearing anywhere in
|
||||
// the query, not just at the start. Word-boundary matching, not a real
|
||||
// SQL parser -- same tradeoffs as the Phase 0/1 version this replaces.
|
||||
var disallowedKeyword = regexp.MustCompile(`(?i)\b(insert|update|delete|alter|drop|truncate|create|grant|revoke|attach|detach|rename|kill|optimize|system|set|exchange|watch)\b`)
|
||||
|
||||
func validateSelectOnly(sql string) error {
|
||||
trimmed := strings.TrimSpace(sql)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("query must not be empty")
|
||||
}
|
||||
|
||||
trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, ";"))
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("query must not be empty")
|
||||
}
|
||||
if strings.Contains(trimmed, ";") {
|
||||
return fmt.Errorf("only a single statement is allowed")
|
||||
}
|
||||
|
||||
firstWord := strings.ToUpper(strings.Fields(trimmed)[0])
|
||||
if firstWord != "SELECT" {
|
||||
return fmt.Errorf("only SELECT queries are allowed")
|
||||
}
|
||||
|
||||
if disallowedKeyword.MatchString(trimmed) {
|
||||
return fmt.Errorf("query contains a disallowed keyword")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package planner
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
var fixedNow = time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestCompileDetectsSQL(t *testing.T) {
|
||||
plan, err := Compile(`SELECT * FROM logs LIMIT 10`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL == "" {
|
||||
t.Fatalf("expected RawSQL to be set, got plan: %+v", plan)
|
||||
}
|
||||
if plan.RawSQL != "SELECT * FROM logs LIMIT 10" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileDetectsSQLCaseInsensitive(t *testing.T) {
|
||||
plan, err := Compile(`select 1`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "select 1" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRejectsNonSelectSQLKeyword(t *testing.T) {
|
||||
_, err := Compile(`DELETE FROM logs`, SQL, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a non-SELECT statement forced to SQL language")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileExplicitLanguageOverridesAutoDetect(t *testing.T) {
|
||||
// "select" as a bare free-text search term -- would be misdetected
|
||||
// as SQL by the heuristic alone, hence the override.
|
||||
plan, err := Compile(`select`, SPL, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "" {
|
||||
t.Fatalf("expected pipe-syntax compilation, got RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != "select" {
|
||||
t.Fatalf("expected a free-text search for 'select', got %+v", plan.TextSearch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSimpleFilter(t *testing.T) {
|
||||
plan, err := Compile(`service=api`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := ir.FilterPredicate{Field: "service", Op: "=", Value: "api"}
|
||||
if len(plan.Filters) != 1 || plan.Filters[0] != want {
|
||||
t.Fatalf("unexpected filters: %+v, want [%+v]", plan.Filters, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFullPipeline(t *testing.T) {
|
||||
plan, err := Compile(`service=api | where status>=500 | stats count by host | sort -count`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Filters) != 2 {
|
||||
t.Fatalf("expected 2 filters (service=api, status>=500), got %+v", plan.Filters)
|
||||
}
|
||||
if plan.Aggregation == nil || len(plan.Aggregation.Funcs) != 1 || plan.Aggregation.Funcs[0].Alias != "count" {
|
||||
t.Fatalf("unexpected aggregation: %+v", plan.Aggregation)
|
||||
}
|
||||
if len(plan.Aggregation.GroupBy) != 1 || plan.Aggregation.GroupBy[0] != "host" {
|
||||
t.Fatalf("unexpected group by: %+v", plan.Aggregation.GroupBy)
|
||||
}
|
||||
if len(plan.Sort) != 1 || plan.Sort[0].Field != "count" || !plan.Sort[0].Desc {
|
||||
t.Fatalf("unexpected sort: %+v", plan.Sort)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTextSearchWithAggregation(t *testing.T) {
|
||||
plan, err := Compile(`message:"connection refused" | stats count by host`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != `"connection refused"` {
|
||||
t.Fatalf("unexpected text search: %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.Aggregation == nil {
|
||||
t.Fatal("expected an aggregation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileImplicitAndBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 {
|
||||
t.Fatalf("expected 1 combined text predicate, got %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error AND timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error AND timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error or timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error OR timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error OR timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenStructuredFiltersErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api or service=web`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: OR between structured filters isn't supported in Phase 2")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "or") {
|
||||
t.Fatalf("error should mention 'or', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRelativeTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`earliest=-1h`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TimeRange == nil {
|
||||
t.Fatal("expected a TimeRange")
|
||||
}
|
||||
want := fixedNow.Add(-1 * time.Hour)
|
||||
if !plan.TimeRange.From.Equal(want) {
|
||||
t.Fatalf("From = %v, want %v", plan.TimeRange.From, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAbsoluteTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`latest="2026-08-14T00:00:00Z"`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC)
|
||||
if !plan.TimeRange.To.Equal(want) {
|
||||
t.Fatalf("To = %v, want %v", plan.TimeRange.To, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileInvalidAbsoluteTimestampErrors(t *testing.T) {
|
||||
_, err := Compile(`latest="not-a-timestamp"`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an invalid absolute timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSumWithoutFieldErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api | stats sum`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: sum() requires a field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAggAliasDefaultsAndCollisionIsDisambiguated(t *testing.T) {
|
||||
plan, err := Compile(`service=api | stats sum(a), sum(b)`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Aggregation.Funcs) != 2 {
|
||||
t.Fatalf("expected 2 agg funcs, got %+v", plan.Aggregation.Funcs)
|
||||
}
|
||||
if plan.Aggregation.Funcs[0].Alias == plan.Aggregation.Funcs[1].Alias {
|
||||
t.Fatalf("expected distinct aliases, got both %q", plan.Aggregation.Funcs[0].Alias)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileHeadDefaultsLimit(t *testing.T) {
|
||||
plan, err := Compile(`service=api | head`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != defaultLimit || plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTailSetsTailFlag(t *testing.T) {
|
||||
plan, err := Compile(`service=api | tail 5`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != 5 || !plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFieldsProjection(t *testing.T) {
|
||||
plan, err := Compile(`service=api | fields host, message`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Fields) != 2 || plan.Fields[0] != "host" || plan.Fields[1] != "message" {
|
||||
t.Fatalf("unexpected fields: %+v", plan.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileParseErrorPropagates(t *testing.T) {
|
||||
_, err := Compile(`service=api | bogus`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected a parse error to propagate")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user