coffey-labs/cairnobs
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 3: dashboards and alerting

Browse Source 
Saved, shareable multi-panel dashboards (table/line/bar/single-stat
panels via gridstack + uPlot, global + per-panel time range, JSON
export/import) and threshold/absence alert rules with an
ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery.

- New /metadata component: Postgres control-plane store for dashboards,
  panels, notification targets, alert rules/state, and delivery log --
  see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree
  family isn't a fit for this access pattern (needs real row-level
  locking and read-your-writes consistency).
- api/internal/dashboards: dashboard/panel CRUD, pure -- panel query
  execution stays client-side, reusing the existing /query endpoint.
- New /alerting service: rule/target CRUD, a ticker-driven evaluator
  (claim-then-evaluate concurrency control, transactional-outbox
  delivery, query errors and threshold zero-rows never coerced into a
  false transition) and webhook/Slack/PagerDuty delivery with
  retry/backoff. See docs/phase-3-alerting-design.md for the full
  state-machine design and the four correctness properties it
  implements.
- web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts
  list/get/apply, seeding a future Terraform provider's JSON contract.
- hack/alert-load-test: 500 rules against real ClickHouse data, real
  measured results in docs/phase-3-runbook.md.

Five real bugs found by actually running this against a live stack
(documented in the runbook, not just fixed silently): a latent Phase 2
bug where ClickHouse rejected the timestamp format used for
earliest=/latest= queries; a "now" literal token injected into query
text; a GridStack/uPlot layout-timing race; JS's Date.parse being too
lenient to use as a timestamp-detection heuristic; a rule's "enabled"
field silently defaulting to false when omitted; and the evaluator's
claim-batch-size and worker-pool-concurrency defaulting to the same
value, causing 500 concurrently-due rules to take 125s to cycle through
instead of the configured 60s.
This commit is contained in:
jcoffey-dev
2026-08-13 17:29:38 -07:00
parent fb5049a747
commit 9435115ab7
88 changed files with 7463 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/cmd/sentryctl/cmd_alerts.go
+51
View File
@@ -0,0 +1,51 @@
package main
import (
"fmt"
"io"
"os"
)
func cmdAlerts(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, "sentryctl alerts: expected a subcommand (list, get, apply)")
return 1
}
alertingURL, rest := extractAlertingAPIFlag(args[1:], os.Getenv)
switch args[0] {
case "list":
return httpGetJSON(alertingURL, "/rules", stdout, stderr)
case "get":
if len(rest) == 0 {
fmt.Fprintln(stderr, "sentryctl alerts get: missing rule id")
return 1
}
return httpGetJSON(alertingURL, "/rules/"+rest[0], stdout, stderr)
case "apply":
if len(rest) == 0 {
fmt.Fprintln(stderr, "sentryctl alerts apply: missing file path")
return 1
}
// POST /rules accepts the same shape it returns -- a rule
// definition file (query, condition, interval, notification
// target ID) applies directly with no reshaping.
return httpPostFileJSON(alertingURL, "/rules", rest[0], stdout, stderr)
default:
fmt.Fprintf(stderr, "sentryctl alerts: unknown subcommand %q (want list, get, apply)\n", args[0])
return 1
}
}
func extractAlertingAPIFlag(args []string, env func(string) string) (alertingURL string, rest []string) {
alertingURL = resolveAlertingURL(env)
for i := 0; i < len(args); i++ {
if args[i] == "--alerting-api" && i+1 < len(args) {
alertingURL = args[i+1]
i++
continue
}
rest = append(rest, args[i])
}
return alertingURL, rest
}
cli/cmd/sentryctl/cmd_alerts_test.go
+63
View File
@@ -0,0 +1,63 @@
package main
import (
"bytes"
"reflect"
"testing"
)
func TestExtractAlertingAPIFlagDefault(t *testing.T) {
alertingURL, rest := extractAlertingAPIFlag([]string{"rule-1"}, func(string) string { return "" })
if alertingURL != defaultAlertingURL {
t.Fatalf("alertingURL = %q, want default %q", alertingURL, defaultAlertingURL)
}
if !reflect.DeepEqual(rest, []string{"rule-1"}) {
t.Fatalf("rest = %v", rest)
}
}
func TestExtractAlertingAPIFlagOverride(t *testing.T) {
alertingURL, rest := extractAlertingAPIFlag([]string{"--alerting-api", "http://custom:9091", "rule-1"}, func(string) string { return "" })
if alertingURL != "http://custom:9091" {
t.Fatalf("alertingURL = %q", alertingURL)
}
if !reflect.DeepEqual(rest, []string{"rule-1"}) {
t.Fatalf("rest = %v", rest)
}
}
func TestExtractAlertingAPIFlagFromEnv(t *testing.T) {
alertingURL, _ := extractAlertingAPIFlag(nil, func(k string) string {
if k == "SENTRYCTL_ALERTING_API_URL" {
return "http://env-alerting:8081"
}
return ""
})
if alertingURL != "http://env-alerting:8081" {
t.Fatalf("alertingURL = %q", alertingURL)
}
}
func TestCmdAlertsMissingSubcommand(t *testing.T) {
var stdout, stderr bytes.Buffer
code := cmdAlerts(nil, &stdout, &stderr)
if code != 1 {
t.Fatalf("code = %d, want 1", code)
}
}
func TestCmdAlertsApplyMissingFile(t *testing.T) {
var stdout, stderr bytes.Buffer
code := cmdAlerts([]string{"apply"}, &stdout, &stderr)
if code != 1 {
t.Fatalf("code = %d, want 1", code)
}
}
func TestCmdAlertsUnknownSubcommand(t *testing.T) {
var stdout, stderr bytes.Buffer
code := cmdAlerts([]string{"bogus"}, &stdout, &stderr)
if code != 1 {
t.Fatalf("code = %d, want 1", code)
}
}
cli/cmd/sentryctl/cmd_dashboards.go
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"fmt"
"io"
"os"
)
func cmdDashboards(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, "sentryctl dashboards: expected a subcommand (list, get, apply)")
return 1
}
apiURL, rest := extractAPIFlag(args[1:], os.Getenv)
switch args[0] {
case "list":
return httpGetJSON(apiURL, "/dashboards", stdout, stderr)
case "get":
if len(rest) == 0 {
fmt.Fprintln(stderr, "sentryctl dashboards get: missing dashboard id")
return 1
}
return httpGetJSON(apiURL, "/dashboards/"+rest[0], stdout, stderr)
case "apply":
if len(rest) == 0 {
fmt.Fprintln(stderr, "sentryctl dashboards apply: missing file path")
return 1
}
// The import endpoint consumes exactly the shape GET
// /dashboards/{id}/export produces and the web UI's Export JSON
// button downloads -- one JSON contract, three call sites.
return httpPostFileJSON(apiURL, "/dashboards/import", rest[0], stdout, stderr)
default:
fmt.Fprintf(stderr, "sentryctl dashboards: unknown subcommand %q (want list, get, apply)\n", args[0])
return 1
}
}
// extractAPIFlag pulls an optional --api <url> out of args, resolving
// the default the same way parsePingArgs/parseQueryArgs do, and returns
// the remaining positional args. Shared by dashboards and alerts since
// both take an optional --api/--alerting-api override the same way.
func extractAPIFlag(args []string, env func(string) string) (apiURL string, rest []string) {
apiURL = resolveAPIURL(env)
for i := 0; i < len(args); i++ {
if args[i] == "--api" && i+1 < len(args) {
apiURL = args[i+1]
i++
continue
}
rest = append(rest, args[i])
}
return apiURL, rest
}
cli/cmd/sentryctl/cmd_dashboards_test.go
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"bytes"
"reflect"
"testing"
)
func TestExtractAPIFlagDefault(t *testing.T) {
apiURL, rest := extractAPIFlag([]string{"abc123"}, func(string) string { return "" })
if apiURL != defaultAPIURL {
t.Fatalf("apiURL = %q, want default %q", apiURL, defaultAPIURL)
}
if !reflect.DeepEqual(rest, []string{"abc123"}) {
t.Fatalf("rest = %v", rest)
}
}
func TestExtractAPIFlagOverride(t *testing.T) {
apiURL, rest := extractAPIFlag([]string{"--api", "http://custom:9090", "abc123"}, func(string) string { return "" })
if apiURL != "http://custom:9090" {
t.Fatalf("apiURL = %q", apiURL)
}
if !reflect.DeepEqual(rest, []string{"abc123"}) {
t.Fatalf("rest = %v, want [abc123] (flag pair stripped)", rest)
}
}
func TestCmdDashboardsMissingSubcommand(t *testing.T) {
var stdout, stderr bytes.Buffer
code := cmdDashboards(nil, &stdout, &stderr)
if code != 1 {
t.Fatalf("code = %d, want 1", code)
}
}
func TestCmdDashboardsGetMissingID(t *testing.T) {
var stdout, stderr bytes.Buffer
code := cmdDashboards([]string{"get"}, &stdout, &stderr)
if code != 1 {
t.Fatalf("code = %d, want 1", code)
}
}
cli/cmd/sentryctl/cmd_ping.go
+44
View File
@@ -0,0 +1,44 @@
package main
import (
"fmt"
"io"
"net/http"
"os"
"time"
)
// parsePingArgs resolves the api base URL for ping: --api flag wins, then
// $SENTRYCTL_API_URL, then the hardcoded default. Kept pure (env passed in
// as a function) and separate from the HTTP call so it's unit-testable
// without a real environment or server.
func parsePingArgs(args []string, env func(string) string) string {
apiURL := resolveAPIURL(env)
for i := 0; i < len(args); i++ {
if args[i] == "--api" && i+1 < len(args) {
apiURL = args[i+1]
i++
}
}
return apiURL
}
func cmdPing(args []string, stdout, stderr io.Writer) int {
apiURL := parsePingArgs(args, os.Getenv)
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(apiURL + "/healthz")
if err != nil {
fmt.Fprintf(stderr, "ping failed: %v\n", err)
return 1
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Fprintf(stderr, "ping failed: api returned status %d\n", resp.StatusCode)
return 1
}
fmt.Fprintln(stdout, "ok")
return 0
}
cli/cmd/sentryctl/cmd_query.go
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
)
type queryArgs struct {
apiURL string
jsonOut bool
language string
query string
}
// parseQueryArgs is pure (env passed in, no I/O), same testability
// reasoning as parsePingArgs. Non-flag arguments are joined with spaces
// to form the query, so `sentryctl query service=api status=500` (no
// quotes, no shell-special characters) works without requiring users to
// quote every query -- though anything using "|" still needs shell
// quoting regardless, since that's a real shell pipe character otherwise.
func parseQueryArgs(args []string, env func(string) string) queryArgs {
qa := queryArgs{apiURL: resolveAPIURL(env)}
var rest []string
for i := 0; i < len(args); i++ {
switch args[i] {
case "--api":
if i+1 < len(args) {
qa.apiURL = args[i+1]
i++
}
case "--json":
qa.jsonOut = true
case "--language":
if i+1 < len(args) {
qa.language = args[i+1]
i++
}
default:
rest = append(rest, args[i])
}
}
qa.query = strings.Join(rest, " ")
return qa
}
type queryRequestBody struct {
Query string `json:"query"`
Language string `json:"language"`
}
type queryResponseBody struct {
Columns []string `json:"columns"`
Rows [][]any `json:"rows"`
}
func cmdQuery(args []string, stdout, stderr io.Writer) int {
qa := parseQueryArgs(args, os.Getenv)
if strings.TrimSpace(qa.query) == "" {
fmt.Fprintln(stderr, "sentryctl query: missing query string")
return 1
}
reqBody, err := json.Marshal(queryRequestBody{Query: qa.query, Language: qa.language})
if err != nil {
fmt.Fprintf(stderr, "encoding request: %v\n", err)
return 1
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Post(qa.apiURL+"/query", "application/json", bytes.NewReader(reqBody))
if err != nil {
fmt.Fprintf(stderr, "query failed: %v\n", err)
return 1
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Fprintf(stderr, "reading response: %v\n", err)
return 1
}
if resp.StatusCode != http.StatusOK {
var errResp errorResponseBody
if json.Unmarshal(respBody, &errResp) == nil && errResp.Error != "" {
fmt.Fprintf(stderr, "query failed: %s\n", errResp.Error)
} else {
fmt.Fprintf(stderr, "query failed: api returned status %d\n", resp.StatusCode)
}
return 1
}
if qa.jsonOut {
_, _ = stdout.Write(respBody)
fmt.Fprintln(stdout)
return 0
}
var result queryResponseBody
if err := json.Unmarshal(respBody, &result); err != nil {
fmt.Fprintf(stderr, "decoding response: %v\n", err)
return 1
}
printTable(stdout, result.Columns, result.Rows)
return 0
}
cli/cmd/sentryctl/httpclient.go
+72
View File
@@ -0,0 +1,72 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"time"
)
var httpClient = &http.Client{Timeout: 30 * time.Second}
// httpGetJSON GETs path and prints the pretty-printed JSON response to
// stdout, or the error body/status to stderr. Shared by dashboards/alerts
// list and get, which otherwise differ only in path and resource name.
func httpGetJSON(baseURL, path string, stdout, stderr io.Writer) int {
resp, err := httpClient.Get(baseURL + path)
if err != nil {
fmt.Fprintf(stderr, "request failed: %v\n", err)
return 1
}
defer resp.Body.Close()
return printJSONResponse(resp, stdout, stderr)
}
// httpPostFileJSON reads file (a JSON document, e.g. an exported
// dashboard or a rule definition) and POSTs it to path as-is -- no
// reshaping, since the file's shape already matches what the endpoint
// expects (the same JSON the web UI's export button and POST /rules
// produce/accept respectively). This is what makes "apply" the seed of a
// future Terraform provider: one JSON contract, multiple callers.
func httpPostFileJSON(baseURL, path, file string, stdout, stderr io.Writer) int {
body, err := os.ReadFile(file)
if err != nil {
fmt.Fprintf(stderr, "reading %s: %v\n", file, err)
return 1
}
resp, err := httpClient.Post(baseURL+path, "application/json", bytes.NewReader(body))
if err != nil {
fmt.Fprintf(stderr, "request failed: %v\n", err)
return 1
}
defer resp.Body.Close()
return printJSONResponse(resp, stdout, stderr)
}
func printJSONResponse(resp *http.Response, stdout, stderr io.Writer) int {
body, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Fprintf(stderr, "reading response: %v\n", err)
return 1
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
var errResp errorResponseBody
if json.Unmarshal(body, &errResp) == nil && errResp.Error != "" {
fmt.Fprintf(stderr, "request failed: %s\n", errResp.Error)
} else {
fmt.Fprintf(stderr, "request failed: status %d\n", resp.StatusCode)
}
return 1
}
var pretty bytes.Buffer
if json.Indent(&pretty, body, "", " ") == nil {
stdout.Write(pretty.Bytes())
} else {
stdout.Write(body)
}
fmt.Fprintln(stdout)
return 0
}
cli/cmd/sentryctl/main.go
+32 -145
View File
@@ -1,22 +1,23 @@
// Command sentryctl is Sentry's control CLI: "ping" (Phase 0) and
// "query" (Phase 2), which accepts either query syntax and hits the same
// POST /query endpoint the web UI does -- no separate query logic here,
// per the Phase 2 task list's explicit instruction.
// Command sentryctl is Sentry's control CLI. Six subcommands now
// (ping, query, dashboards, alerts) clearly justify splitting dispatch
// across files -- see cli/README.md's "revisit once there's a real
// command tree" note -- while keeping the same hand-rolled switch on
// os.Args, no CLI framework, per that same README.
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"text/tabwriter"
"time"
)
const defaultAPIURL = "http://localhost:8080"
const (
defaultAPIURL = "http://localhost:8080"
defaultAlertingURL = "http://localhost:8081"
)
func main() {
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
@@ -33,6 +34,10 @@ func run(args []string, stdout, stderr io.Writer) int {
return cmdPing(args[1:], stdout, stderr)
case "query":
return cmdQuery(args[1:], stdout, stderr)
case "dashboards":
return cmdDashboards(args[1:], stdout, stderr)
case "alerts":
return cmdAlerts(args[1:], stdout, stderr)
case "-h", "--help", "help":
usage(stdout)
return 0
@@ -49,15 +54,25 @@ func usage(w io.Writer) {
Usage:
sentryctl ping [--api <url>]
sentryctl query "<query>" [--api <url>] [--language sql|spl] [--json]
sentryctl dashboards list|get <id>|apply <file> [--api <url>]
sentryctl alerts list|get <id>|apply <file> [--alerting-api <url>]
Commands:
ping Checks that the api service is reachable via GET /healthz.
query Runs a query (pipe syntax or SQL) against POST /query and
prints the result as a table, or as JSON with --json. Quote
the query in your shell -- pipe syntax uses "|", which your
shell will otherwise interpret itself.
ping Checks that the api service is reachable via GET /healthz.
query Runs a query (pipe syntax or SQL) against POST /query and
prints the result as a table, or as JSON with --json. Quote
the query in your shell -- pipe syntax uses "|", which your
shell will otherwise interpret itself.
dashboards list/get/apply against api's dashboard CRUD endpoints.
"apply <file>" imports a dashboard exported via the web
UI's Export JSON button or GET /dashboards/{id}/export --
the same JSON shape both places, Terraform-friendly.
alerts list/get/apply against alerting's rule CRUD endpoints.
"apply <file>" creates a rule from a JSON file with the
same shape POST /rules accepts.
--api defaults to $SENTRYCTL_API_URL, or `+defaultAPIURL+` if unset.
--alerting-api defaults to $SENTRYCTL_ALERTING_API_URL, or `+defaultAlertingURL+` if unset.
--language overrides auto-detection; omit it for the common case.`)
}
@@ -68,145 +83,17 @@ func resolveAPIURL(env func(string) string) string {
return defaultAPIURL
}
// parsePingArgs resolves the api base URL for ping: --api flag wins, then
// $SENTRYCTL_API_URL, then the hardcoded default. Kept pure (env passed in
// as a function) and separate from the HTTP call so it's unit-testable
// without a real environment or server.
func parsePingArgs(args []string, env func(string) string) string {
apiURL := resolveAPIURL(env)
for i := 0; i < len(args); i++ {
if args[i] == "--api" && i+1 < len(args) {
apiURL = args[i+1]
i++
}
func resolveAlertingURL(env func(string) string) string {
if v := env("SENTRYCTL_ALERTING_API_URL"); v != "" {
return v
}
return apiURL
}
func cmdPing(args []string, stdout, stderr io.Writer) int {
apiURL := parsePingArgs(args, os.Getenv)
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(apiURL + "/healthz")
if err != nil {
fmt.Fprintf(stderr, "ping failed: %v\n", err)
return 1
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Fprintf(stderr, "ping failed: api returned status %d\n", resp.StatusCode)
return 1
}
fmt.Fprintln(stdout, "ok")
return 0
}
type queryArgs struct {
apiURL string
jsonOut bool
language string
query string
}
// parseQueryArgs is pure (env passed in, no I/O), same testability
// reasoning as parsePingArgs. Non-flag arguments are joined with spaces
// to form the query, so `sentryctl query service=api status=500` (no
// quotes, no shell-special characters) works without requiring users to
// quote every query -- though anything using "|" still needs shell
// quoting regardless, since that's a real shell pipe character otherwise.
func parseQueryArgs(args []string, env func(string) string) queryArgs {
qa := queryArgs{apiURL: resolveAPIURL(env)}
var rest []string
for i := 0; i < len(args); i++ {
switch args[i] {
case "--api":
if i+1 < len(args) {
qa.apiURL = args[i+1]
i++
}
case "--json":
qa.jsonOut = true
case "--language":
if i+1 < len(args) {
qa.language = args[i+1]
i++
}
default:
rest = append(rest, args[i])
}
}
qa.query = strings.Join(rest, " ")
return qa
}
type queryRequestBody struct {
Query string `json:"query"`
Language string `json:"language"`
}
type queryResponseBody struct {
Columns []string `json:"columns"`
Rows [][]any `json:"rows"`
return defaultAlertingURL
}
type errorResponseBody struct {
Error string `json:"error"`
}
func cmdQuery(args []string, stdout, stderr io.Writer) int {
qa := parseQueryArgs(args, os.Getenv)
if strings.TrimSpace(qa.query) == "" {
fmt.Fprintln(stderr, "sentryctl query: missing query string")
return 1
}
reqBody, err := json.Marshal(queryRequestBody{Query: qa.query, Language: qa.language})
if err != nil {
fmt.Fprintf(stderr, "encoding request: %v\n", err)
return 1
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Post(qa.apiURL+"/query", "application/json", bytes.NewReader(reqBody))
if err != nil {
fmt.Fprintf(stderr, "query failed: %v\n", err)
return 1
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Fprintf(stderr, "reading response: %v\n", err)
return 1
}
if resp.StatusCode != http.StatusOK {
var errResp errorResponseBody
if json.Unmarshal(respBody, &errResp) == nil && errResp.Error != "" {
fmt.Fprintf(stderr, "query failed: %s\n", errResp.Error)
} else {
fmt.Fprintf(stderr, "query failed: api returned status %d\n", resp.StatusCode)
}
return 1
}
if qa.jsonOut {
_, _ = stdout.Write(respBody)
fmt.Fprintln(stdout)
return 0
}
var result queryResponseBody
if err := json.Unmarshal(respBody, &result); err != nil {
fmt.Fprintf(stderr, "decoding response: %v\n", err)
return 1
}
printTable(stdout, result.Columns, result.Rows)
return 0
}
func printTable(w io.Writer, columns []string, rows [][]any) {
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, strings.Join(columns, "\t"))