cairnobs/api/internal/querylang/lexer/lexer.go

// Package lexer tokenizes the pipe-syntax query language. Deliberately
// simple: keywords (where/stats/sort/and/etc.) aren't distinct token
// kinds -- they're just Ident tokens whose value the parser checks
// against a keyword set, so the lexer stays context-free and the parser
// owns all the grammar decisions. See /docs/query-language-design.md.
package lexer

import "fmt"

type Kind int

const (
	EOF Kind = iota
	Illegal
	Ident  // bare words: field names, keywords, unquoted values/free-text terms
	String // quoted string: "..."
	Number // 123, 1.5
	Pipe   // |
	Eq     // =
	Neq    // !=
	Gt     // >
	Gte    // >=
	Lt     // <
	Lte    // <=
	Colon  // :
	Comma  // ,
	LParen // (
	RParen // )
	Minus  // -
	Plus   // +
	Star   // * (only meaningful inside count(*), same as SQL)
)

type Token struct {
	Kind  Kind
	Value string
	Pos   int // byte offset into the original input, for error messages
}

func (t Token) String() string {
	return fmt.Sprintf("%s(%q)@%d", t.Kind, t.Value, t.Pos)
}

func (k Kind) String() string {
	switch k {
	case EOF:
		return "EOF"
	case Illegal:
		return "ILLEGAL"
	case Ident:
		return "IDENT"
	case String:
		return "STRING"
	case Number:
		return "NUMBER"
	case Pipe:
		return "PIPE"
	case Eq:
		return "EQ"
	case Neq:
		return "NEQ"
	case Gt:
		return "GT"
	case Gte:
		return "GTE"
	case Lt:
		return "LT"
	case Lte:
		return "LTE"
	case Colon:
		return "COLON"
	case Comma:
		return "COMMA"
	case LParen:
		return "LPAREN"
	case RParen:
		return "RPAREN"
	case Minus:
		return "MINUS"
	case Plus:
		return "PLUS"
	case Star:
		return "STAR"
	default:
		return "UNKNOWN"
	}
}

type Lexer struct {
	input []rune
	pos   int
}

func New(input string) *Lexer {
	return &Lexer{input: []rune(input)}
}

func (l *Lexer) Next() Token {
	l.skipWhitespace()
	if l.pos >= len(l.input) {
		return Token{Kind: EOF, Pos: l.pos}
	}

	start := l.pos
	c := l.input[l.pos]

	switch {
	case c == '|':
		l.pos++
		return Token{Kind: Pipe, Value: "|", Pos: start}
	case c == '=':
		l.pos++
		return Token{Kind: Eq, Value: "=", Pos: start}
	case c == '!' && l.peek(1) == '=':
		l.pos += 2
		return Token{Kind: Neq, Value: "!=", Pos: start}
	case c == '>' && l.peek(1) == '=':
		l.pos += 2
		return Token{Kind: Gte, Value: ">=", Pos: start}
	case c == '>':
		l.pos++
		return Token{Kind: Gt, Value: ">", Pos: start}
	case c == '<' && l.peek(1) == '=':
		l.pos += 2
		return Token{Kind: Lte, Value: "<=", Pos: start}
	case c == '<':
		l.pos++
		return Token{Kind: Lt, Value: "<", Pos: start}
	case c == ':':
		l.pos++
		return Token{Kind: Colon, Value: ":", Pos: start}
	case c == ',':
		l.pos++
		return Token{Kind: Comma, Value: ",", Pos: start}
	case c == '(':
		l.pos++
		return Token{Kind: LParen, Value: "(", Pos: start}
	case c == ')':
		l.pos++
		return Token{Kind: RParen, Value: ")", Pos: start}
	case c == '-':
		l.pos++
		return Token{Kind: Minus, Value: "-", Pos: start}
	case c == '+':
		l.pos++
		return Token{Kind: Plus, Value: "+", Pos: start}
	case c == '*':
		l.pos++
		return Token{Kind: Star, Value: "*", Pos: start}
	case c == '"':
		return l.lexString()
	case isDigit(c):
		return l.lexNumber()
	case isIdentStart(c):
		return l.lexIdent()
	default:
		l.pos++
		return Token{Kind: Illegal, Value: string(c), Pos: start}
	}
}

func (l *Lexer) peek(offset int) rune {
	p := l.pos + offset
	if p >= len(l.input) {
		return 0
	}
	return l.input[p]
}

func (l *Lexer) skipWhitespace() {
	for l.pos < len(l.input) {
		switch l.input[l.pos] {
		case ' ', '\t', '\n', '\r':
			l.pos++
		default:
			return
		}
	}
}

func (l *Lexer) lexString() Token {
	start := l.pos
	l.pos++ // consume opening quote
	var sb []rune
	for l.pos < len(l.input) {
		c := l.input[l.pos]
		if c == '"' {
			l.pos++
			return Token{Kind: String, Value: string(sb), Pos: start}
		}
		if c == '\\' && l.pos+1 < len(l.input) {
			l.pos++
			sb = append(sb, l.input[l.pos])
			l.pos++
			continue
		}
		sb = append(sb, c)
		l.pos++
	}
	// unterminated string -- return what we have as Illegal so the
	// parser can produce a clear "unterminated string" error rather than
	// the lexer silently accepting it.
	return Token{Kind: Illegal, Value: string(sb), Pos: start}
}

func (l *Lexer) lexNumber() Token {
	start := l.pos
	for l.pos < len(l.input) && isDigit(l.input[l.pos]) {
		l.pos++
	}
	if l.pos < len(l.input) && l.input[l.pos] == '.' && l.pos+1 < len(l.input) && isDigit(l.input[l.pos+1]) {
		l.pos++
		for l.pos < len(l.input) && isDigit(l.input[l.pos]) {
			l.pos++
		}
	}
	return Token{Kind: Number, Value: string(l.input[start:l.pos]), Pos: start}
}

func (l *Lexer) lexIdent() Token {
	start := l.pos
	for l.pos < len(l.input) && isIdentPart(l.input[l.pos]) {
		l.pos++
	}
	return Token{Kind: Ident, Value: string(l.input[start:l.pos]), Pos: start}
}

func isDigit(c rune) bool { return c >= '0' && c <= '9' }

func isIdentStart(c rune) bool {
	return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

// Identifiers allow dots (e.g. winevt.event_id, a real attribute key
// shape from Phase 1), digits, and internal hyphens (e.g. host-03,
// api-service -- real, common bare-word values with no need for
// quoting) after the first character. A leading hyphen is deliberately
// NOT part of isIdentStart -- Minus has to stay its own token there so
// `earliest=-1h` and `sort -count`'s leading sign still lex correctly;
// this only affects a hyphen once a token has already started with a
// real identifier character.
func isIdentPart(c rune) bool {
	return isIdentStart(c) || isDigit(c) || c == '.' || c == '_' || c == '-'
}