package lexer import "testing" func collectKinds(input string) []Kind { l := New(input) var kinds []Kind for { tok := l.Next() kinds = append(kinds, tok.Kind) if tok.Kind == EOF { return kinds } } } func TestLexSimpleFilter(t *testing.T) { got := collectKinds(`service=api`) want := []Kind{Ident, Eq, Ident, EOF} assertKinds(t, got, want) } func TestLexPipeline(t *testing.T) { got := collectKinds(`service=api | where status>=500 | stats count(*) by host`) want := []Kind{ Ident, Eq, Ident, Pipe, Ident, Ident, Gte, Number, Pipe, Ident, Ident, LParen, Star, RParen, Ident, Ident, EOF, } assertKinds(t, got, want) } func TestLexOperators(t *testing.T) { got := collectKinds(`= != > >= < <=`) want := []Kind{Eq, Neq, Gt, Gte, Lt, Lte, EOF} assertKinds(t, got, want) } func TestLexQuotedString(t *testing.T) { l := New(`"connection refused"`) tok := l.Next() if tok.Kind != String { t.Fatalf("Kind = %v, want String", tok.Kind) } if tok.Value != "connection refused" { t.Fatalf("Value = %q, want %q", tok.Value, "connection refused") } } func TestLexQuotedStringWithEscapes(t *testing.T) { l := New(`"has \"quotes\" and \\backslash"`) tok := l.Next() if tok.Kind != String { t.Fatalf("Kind = %v, want String", tok.Kind) } want := `has "quotes" and \backslash` if tok.Value != want { t.Fatalf("Value = %q, want %q", tok.Value, want) } } func TestLexUnterminatedStringIsIllegal(t *testing.T) { l := New(`"unterminated`) tok := l.Next() if tok.Kind != Illegal { t.Fatalf("Kind = %v, want Illegal", tok.Kind) } } func TestLexFieldWithDots(t *testing.T) { l := New(`winevt.event_id=4625`) tok := l.Next() if tok.Kind != Ident || tok.Value != "winevt.event_id" { t.Fatalf("got %v, want Ident(winevt.event_id)", tok) } } // TestLexIdentWithInternalHyphens is a regression test for a real bug: // isIdentPart didn't include '-', so a bare (unquoted) hyphenated value // like host-03 -- the exact example /docs/query-language-reference.md // itself uses (`host!=host-03`) -- lexed as IDENT("host") MINUS // IDENT("03") and failed to parse at all. A leading hyphen must still // lex as its own MINUS token (earliest=-1h, sort -count depend on it) -- // only an internal hyphen, once a real identifier character has already // started the token, should be absorbed. func TestLexIdentWithInternalHyphens(t *testing.T) { cases := []string{"host-03", "api-service", "heartbeat-test-host", "multi-hyphen-value"} for _, c := range cases { l := New(c) tok := l.Next() if tok.Kind != Ident || tok.Value != c { t.Errorf("lexing %q: got %v, want Ident(%s)", c, tok, c) } if end := l.Next(); end.Kind != EOF { t.Errorf("lexing %q: expected EOF after the identifier, got %v", c, end) } } } func TestLexFilterWithHyphenatedValue(t *testing.T) { got := collectKinds(`host!=host-03`) want := []Kind{Ident, Neq, Ident, EOF} assertKinds(t, got, want) } func TestLexNumber(t *testing.T) { cases := []string{"123", "1.5", "0"} for _, c := range cases { l := New(c) tok := l.Next() if tok.Kind != Number || tok.Value != c { t.Errorf("lexing %q: got %v, want Number(%s)", c, tok, c) } } } func TestLexNegativeTimeExpr(t *testing.T) { // "-1h" lexes as MINUS, NUMBER, IDENT -- the parser composes these, // not the lexer (see package doc comment). got := collectKinds(`-1h`) want := []Kind{Minus, Number, Ident, EOF} assertKinds(t, got, want) } func TestLexWhitespaceIsSkipped(t *testing.T) { got := collectKinds(" service = api ") want := []Kind{Ident, Eq, Ident, EOF} assertKinds(t, got, want) } func TestLexEmptyInput(t *testing.T) { got := collectKinds("") want := []Kind{EOF} assertKinds(t, got, want) } func TestLexIllegalCharacter(t *testing.T) { l := New(`$`) tok := l.Next() if tok.Kind != Illegal { t.Fatalf("Kind = %v, want Illegal", tok.Kind) } } func TestTokenPositionsAreByteOffsets(t *testing.T) { l := New(`service=api`) first := l.Next() second := l.Next() if first.Pos != 0 { t.Errorf("first.Pos = %d, want 0", first.Pos) } if second.Pos != 7 { t.Errorf("second.Pos = %d, want 7", second.Pos) } } func assertKinds(t *testing.T, got, want []Kind) { t.Helper() if len(got) != len(want) { t.Fatalf("got %d tokens %v, want %d tokens %v", len(got), got, len(want), want) } for i := range got { if got[i] != want[i] { t.Fatalf("token %d: got %v, want %v (full: got=%v want=%v)", i, got[i], want[i], got, want) } } }