Phase 2: unified query language spanning ClickHouse and Tantivy
Replaces the separate SQL-only /query and text-only /search endpoints with one pipe-syntax query language (plus raw SQL escape hatch) that compiles to a single IR and execution plan across both backends, so a query like `message:"connection refused" | stats count by host` runs as one request instead of two disjoint tools. - api/internal/querylang: lexer -> ast -> parser -> ir -> planner -> executor, each layer independently tested. - Execution generalizes Phase 1's proven Tantivy-prefilter pattern into a 4-way routing table (pure ClickHouse / text-only / text + aggregation / raw SQL passthrough). - Unified web query page and `sentryctl query`, both hitting the same POST /query endpoint. - Benchmarked against a real 1,022,000-row dataset (hack/benchmark-fixture); caught and fixed a real bug where the Tantivy prefilter cap (10,000) produced an IN-clause exceeding ClickHouse's default max_query_size -- lowered to 5,000, documented in docs/query-language-design.md and docs/phase-2-runbook.md. - docs/query-language-reference.md: customer-facing syntax reference.
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
package planner
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sentry/sentry/api/internal/querylang/ir"
|
||||
)
|
||||
|
||||
var fixedNow = time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestCompileDetectsSQL(t *testing.T) {
|
||||
plan, err := Compile(`SELECT * FROM logs LIMIT 10`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL == "" {
|
||||
t.Fatalf("expected RawSQL to be set, got plan: %+v", plan)
|
||||
}
|
||||
if plan.RawSQL != "SELECT * FROM logs LIMIT 10" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileDetectsSQLCaseInsensitive(t *testing.T) {
|
||||
plan, err := Compile(`select 1`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "select 1" {
|
||||
t.Fatalf("RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRejectsNonSelectSQLKeyword(t *testing.T) {
|
||||
_, err := Compile(`DELETE FROM logs`, SQL, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a non-SELECT statement forced to SQL language")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileExplicitLanguageOverridesAutoDetect(t *testing.T) {
|
||||
// "select" as a bare free-text search term -- would be misdetected
|
||||
// as SQL by the heuristic alone, hence the override.
|
||||
plan, err := Compile(`select`, SPL, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.RawSQL != "" {
|
||||
t.Fatalf("expected pipe-syntax compilation, got RawSQL = %q", plan.RawSQL)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != "select" {
|
||||
t.Fatalf("expected a free-text search for 'select', got %+v", plan.TextSearch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSimpleFilter(t *testing.T) {
|
||||
plan, err := Compile(`service=api`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := ir.FilterPredicate{Field: "service", Op: "=", Value: "api"}
|
||||
if len(plan.Filters) != 1 || plan.Filters[0] != want {
|
||||
t.Fatalf("unexpected filters: %+v, want [%+v]", plan.Filters, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFullPipeline(t *testing.T) {
|
||||
plan, err := Compile(`service=api | where status>=500 | stats count by host | sort -count`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Filters) != 2 {
|
||||
t.Fatalf("expected 2 filters (service=api, status>=500), got %+v", plan.Filters)
|
||||
}
|
||||
if plan.Aggregation == nil || len(plan.Aggregation.Funcs) != 1 || plan.Aggregation.Funcs[0].Alias != "count" {
|
||||
t.Fatalf("unexpected aggregation: %+v", plan.Aggregation)
|
||||
}
|
||||
if len(plan.Aggregation.GroupBy) != 1 || plan.Aggregation.GroupBy[0] != "host" {
|
||||
t.Fatalf("unexpected group by: %+v", plan.Aggregation.GroupBy)
|
||||
}
|
||||
if len(plan.Sort) != 1 || plan.Sort[0].Field != "count" || !plan.Sort[0].Desc {
|
||||
t.Fatalf("unexpected sort: %+v", plan.Sort)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTextSearchWithAggregation(t *testing.T) {
|
||||
plan, err := Compile(`message:"connection refused" | stats count by host`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 || plan.TextSearch[0].Query != `"connection refused"` {
|
||||
t.Fatalf("unexpected text search: %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.Aggregation == nil {
|
||||
t.Fatal("expected an aggregation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileImplicitAndBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.TextSearch) != 1 {
|
||||
t.Fatalf("expected 1 combined text predicate, got %+v", plan.TextSearch)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error AND timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error AND timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenFreeTextTerms(t *testing.T) {
|
||||
plan, err := Compile(`error or timeout`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TextSearch[0].Query != "error OR timeout" {
|
||||
t.Fatalf("Query = %q, want %q", plan.TextSearch[0].Query, "error OR timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileOrBetweenStructuredFiltersErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api or service=web`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: OR between structured filters isn't supported in Phase 2")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "or") {
|
||||
t.Fatalf("error should mention 'or', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRelativeTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`earliest=-1h`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.TimeRange == nil {
|
||||
t.Fatal("expected a TimeRange")
|
||||
}
|
||||
want := fixedNow.Add(-1 * time.Hour)
|
||||
if !plan.TimeRange.From.Equal(want) {
|
||||
t.Fatalf("From = %v, want %v", plan.TimeRange.From, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAbsoluteTimeBound(t *testing.T) {
|
||||
plan, err := Compile(`latest="2026-08-14T00:00:00Z"`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
want := time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC)
|
||||
if !plan.TimeRange.To.Equal(want) {
|
||||
t.Fatalf("To = %v, want %v", plan.TimeRange.To, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileInvalidAbsoluteTimestampErrors(t *testing.T) {
|
||||
_, err := Compile(`latest="not-a-timestamp"`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an invalid absolute timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileSumWithoutFieldErrors(t *testing.T) {
|
||||
_, err := Compile(`service=api | stats sum`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: sum() requires a field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileAggAliasDefaultsAndCollisionIsDisambiguated(t *testing.T) {
|
||||
plan, err := Compile(`service=api | stats sum(a), sum(b)`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Aggregation.Funcs) != 2 {
|
||||
t.Fatalf("expected 2 agg funcs, got %+v", plan.Aggregation.Funcs)
|
||||
}
|
||||
if plan.Aggregation.Funcs[0].Alias == plan.Aggregation.Funcs[1].Alias {
|
||||
t.Fatalf("expected distinct aliases, got both %q", plan.Aggregation.Funcs[0].Alias)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileHeadDefaultsLimit(t *testing.T) {
|
||||
plan, err := Compile(`service=api | head`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != defaultLimit || plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileTailSetsTailFlag(t *testing.T) {
|
||||
plan, err := Compile(`service=api | tail 5`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if plan.Limit == nil || plan.Limit.N != 5 || !plan.Limit.Tail {
|
||||
t.Fatalf("unexpected limit: %+v", plan.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileFieldsProjection(t *testing.T) {
|
||||
plan, err := Compile(`service=api | fields host, message`, Auto, fixedNow)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile() error = %v", err)
|
||||
}
|
||||
if len(plan.Fields) != 2 || plan.Fields[0] != "host" || plan.Fields[1] != "message" {
|
||||
t.Fatalf("unexpected fields: %+v", plan.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileParseErrorPropagates(t *testing.T) {
|
||||
_, err := Compile(`service=api | bogus`, Auto, fixedNow)
|
||||
if err == nil {
|
||||
t.Fatal("expected a parse error to propagate")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user