Saved, shareable multi-panel dashboards (table/line/bar/single-stat panels via gridstack + uPlot, global + per-panel time range, JSON export/import) and threshold/absence alert rules with an ok/pending/firing evaluator and webhook/Slack/PagerDuty delivery. - New /metadata component: Postgres control-plane store for dashboards, panels, notification targets, alert rules/state, and delivery log -- see docs/phase-3-dashboard-design.md for why ClickHouse's MergeTree family isn't a fit for this access pattern (needs real row-level locking and read-your-writes consistency). - api/internal/dashboards: dashboard/panel CRUD, pure -- panel query execution stays client-side, reusing the existing /query endpoint. - New /alerting service: rule/target CRUD, a ticker-driven evaluator (claim-then-evaluate concurrency control, transactional-outbox delivery, query errors and threshold zero-rows never coerced into a false transition) and webhook/Slack/PagerDuty delivery with retry/backoff. See docs/phase-3-alerting-design.md for the full state-machine design and the four correctness properties it implements. - web: /dashboards and /alerts UIs; cli: sentryctl dashboards/alerts list/get/apply, seeding a future Terraform provider's JSON contract. - hack/alert-load-test: 500 rules against real ClickHouse data, real measured results in docs/phase-3-runbook.md. Five real bugs found by actually running this against a live stack (documented in the runbook, not just fixed silently): a latent Phase 2 bug where ClickHouse rejected the timestamp format used for earliest=/latest= queries; a "now" literal token injected into query text; a GridStack/uPlot layout-timing race; JS's Date.parse being too lenient to use as a timestamp-detection heuristic; a rule's "enabled" field silently defaulting to false when omitted; and the evaluator's claim-batch-size and worker-pool-concurrency defaulting to the same value, causing 500 concurrently-due rules to take 125s to cycle through instead of the configured 60s.
262 lines
8.6 KiB
Go
262 lines
8.6 KiB
Go
// Package httpapi is alerting's REST surface: rule CRUD, notification
|
|
// target CRUD, and a read-only delivery log -- mirrors
|
|
// api/internal/dashboards' Handler/Store shape (narrow interfaces,
|
|
// pgx-backed production implementation, fakes in tests).
|
|
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"net/http"
|
|
|
|
"github.com/sentry/sentry/alerting/internal/notifystore"
|
|
"github.com/sentry/sentry/alerting/internal/rulestore"
|
|
)
|
|
|
|
type ruleStore interface {
|
|
Create(ctx context.Context, r *rulestore.Rule) error
|
|
List(ctx context.Context) ([]rulestore.RuleWithState, error)
|
|
Get(ctx context.Context, id string) (*rulestore.RuleWithState, error)
|
|
Delete(ctx context.Context, id string) error
|
|
}
|
|
|
|
type targetStore interface {
|
|
Create(ctx context.Context, t *notifystore.Target) error
|
|
List(ctx context.Context) ([]notifystore.Target, error)
|
|
Get(ctx context.Context, id string) (*notifystore.Target, error)
|
|
Delete(ctx context.Context, id string) error
|
|
}
|
|
|
|
type deliveryReader interface {
|
|
ListForRule(ctx context.Context, ruleID string, limit int) ([]rulestore.DeliveryLogEntry, error)
|
|
}
|
|
|
|
type Handler struct {
|
|
logger *slog.Logger
|
|
rules ruleStore
|
|
targets targetStore
|
|
deliveries deliveryReader
|
|
}
|
|
|
|
func NewHandler(logger *slog.Logger, rules ruleStore, targets targetStore, deliveries deliveryReader) *Handler {
|
|
return &Handler{logger: logger, rules: rules, targets: targets, deliveries: deliveries}
|
|
}
|
|
|
|
func (h *Handler) RegisterRoutes(mux *http.ServeMux) {
|
|
mux.HandleFunc("GET /healthz", h.handleHealthz)
|
|
|
|
mux.HandleFunc("POST /rules", h.handleCreateRule)
|
|
mux.HandleFunc("GET /rules", h.handleListRules)
|
|
mux.HandleFunc("GET /rules/{id}", h.handleGetRule)
|
|
mux.HandleFunc("DELETE /rules/{id}", h.handleDeleteRule)
|
|
mux.HandleFunc("GET /rules/{id}/deliveries", h.handleListDeliveries)
|
|
|
|
mux.HandleFunc("POST /targets", h.handleCreateTarget)
|
|
mux.HandleFunc("GET /targets", h.handleListTargets)
|
|
mux.HandleFunc("GET /targets/{id}", h.handleGetTarget)
|
|
mux.HandleFunc("DELETE /targets/{id}", h.handleDeleteTarget)
|
|
}
|
|
|
|
const maxBodyBytes = 1 << 20 // 1 MiB, same cap as api/internal/queryapi and dashboards
|
|
|
|
func (h *Handler) handleHealthz(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|
|
|
|
// createRuleRequest mirrors rulestore.Rule except Enabled is a pointer:
|
|
// a plain bool can't distinguish "omitted" from "explicitly false" on
|
|
// decode, and Go's zero value for bool is false -- without this, a
|
|
// create request that simply doesn't mention "enabled" would silently
|
|
// create a rule the evaluator's claim query never picks up (found by
|
|
// actually creating a rule through this endpoint and checking the
|
|
// response). Omitted means enabled; only an explicit "enabled": false
|
|
// creates a disabled rule.
|
|
type createRuleRequest struct {
|
|
rulestore.Rule
|
|
Enabled *bool `json:"enabled"`
|
|
}
|
|
|
|
func (h *Handler) handleCreateRule(w http.ResponseWriter, r *http.Request) {
|
|
var req createRuleRequest
|
|
if !decodeJSON(w, r, &req) {
|
|
return
|
|
}
|
|
rule := req.Rule
|
|
rule.Enabled = req.Enabled == nil || *req.Enabled
|
|
if err := validateRule(&rule); err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
if err := h.rules.Create(r.Context(), &rule); err != nil {
|
|
h.logger.Error("creating rule", "error", err)
|
|
writeError(w, http.StatusInternalServerError, "creating rule failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusCreated, rule)
|
|
}
|
|
|
|
func (h *Handler) handleListRules(w http.ResponseWriter, r *http.Request) {
|
|
rules, err := h.rules.List(r.Context())
|
|
if err != nil {
|
|
h.logger.Error("listing rules", "error", err)
|
|
writeError(w, http.StatusInternalServerError, "listing rules failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, rules)
|
|
}
|
|
|
|
func (h *Handler) handleGetRule(w http.ResponseWriter, r *http.Request) {
|
|
rule, err := h.rules.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
h.writeStoreErr(w, err, "fetching rule")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, rule)
|
|
}
|
|
|
|
func (h *Handler) handleDeleteRule(w http.ResponseWriter, r *http.Request) {
|
|
if err := h.rules.Delete(r.Context(), r.PathValue("id")); err != nil {
|
|
h.writeStoreErr(w, err, "deleting rule")
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
func (h *Handler) handleListDeliveries(w http.ResponseWriter, r *http.Request) {
|
|
entries, err := h.deliveries.ListForRule(r.Context(), r.PathValue("id"), 100)
|
|
if err != nil {
|
|
h.logger.Error("listing deliveries", "rule_id", r.PathValue("id"), "error", err)
|
|
writeError(w, http.StatusInternalServerError, "listing deliveries failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, entries)
|
|
}
|
|
|
|
func (h *Handler) handleCreateTarget(w http.ResponseWriter, r *http.Request) {
|
|
var target notifystore.Target
|
|
if !decodeJSON(w, r, &target) {
|
|
return
|
|
}
|
|
if target.Name == "" {
|
|
writeError(w, http.StatusBadRequest, "name must not be empty")
|
|
return
|
|
}
|
|
if !notifystore.ValidKind(target.Kind) {
|
|
writeError(w, http.StatusBadRequest, "kind must be one of webhook, slack, pagerduty")
|
|
return
|
|
}
|
|
if target.WebhookURL == "" {
|
|
writeError(w, http.StatusBadRequest, "webhook_url must not be empty")
|
|
return
|
|
}
|
|
if err := h.targets.Create(r.Context(), &target); err != nil {
|
|
h.logger.Error("creating notification target", "error", err)
|
|
writeError(w, http.StatusInternalServerError, "creating notification target failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusCreated, target)
|
|
}
|
|
|
|
func (h *Handler) handleListTargets(w http.ResponseWriter, r *http.Request) {
|
|
targets, err := h.targets.List(r.Context())
|
|
if err != nil {
|
|
h.logger.Error("listing notification targets", "error", err)
|
|
writeError(w, http.StatusInternalServerError, "listing notification targets failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, targets)
|
|
}
|
|
|
|
func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) {
|
|
target, err := h.targets.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
h.writeStoreErr(w, err, "fetching notification target")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, target)
|
|
}
|
|
|
|
func (h *Handler) handleDeleteTarget(w http.ResponseWriter, r *http.Request) {
|
|
if err := h.targets.Delete(r.Context(), r.PathValue("id")); err != nil {
|
|
h.writeStoreErr(w, err, "deleting notification target")
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
// validateRule enforces the shape the evaluator assumes:
|
|
// condition_type-appropriate fields present, a sane interval floor.
|
|
// Mirrors the DB CHECK constraints so bad input is rejected with a
|
|
// clear message here rather than surfacing as an opaque constraint
|
|
// violation from Postgres.
|
|
func validateRule(r *rulestore.Rule) error {
|
|
if r.Name == "" {
|
|
return errBadRequest("name must not be empty")
|
|
}
|
|
if r.Query == "" {
|
|
return errBadRequest("query must not be empty")
|
|
}
|
|
if r.NotificationTargetID == "" {
|
|
return errBadRequest("notification_target_id must not be empty")
|
|
}
|
|
if r.EvalIntervalSeconds < 30 {
|
|
return errBadRequest("eval_interval_seconds must be at least 30")
|
|
}
|
|
switch r.ConditionType {
|
|
case rulestore.ConditionThreshold:
|
|
if r.Comparator == nil || !rulestore.ValidComparator(*r.Comparator) {
|
|
return errBadRequest("threshold rules require a valid comparator (gt, gte, lt, lte, eq, ne)")
|
|
}
|
|
if r.ThresholdValue == nil {
|
|
return errBadRequest("threshold rules require threshold_value")
|
|
}
|
|
case rulestore.ConditionAbsence:
|
|
// No comparator/threshold_value needed -- absence is "the query
|
|
// returned zero rows in its own earliest=/latest= window."
|
|
default:
|
|
return errBadRequest("condition_type must be \"threshold\" or \"absence\"")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type badRequestError string
|
|
|
|
func (e badRequestError) Error() string { return string(e) }
|
|
func errBadRequest(msg string) error { return badRequestError(msg) }
|
|
|
|
func (h *Handler) writeStoreErr(w http.ResponseWriter, err error, action string) {
|
|
if errors.Is(err, rulestore.ErrNotFound) || errors.Is(err, notifystore.ErrNotFound) {
|
|
writeError(w, http.StatusNotFound, "not found")
|
|
return
|
|
}
|
|
h.logger.Error(action, "error", err)
|
|
writeError(w, http.StatusInternalServerError, action+" failed")
|
|
}
|
|
|
|
func decodeJSON(w http.ResponseWriter, r *http.Request, v any) bool {
|
|
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
|
if err := json.NewDecoder(r.Body).Decode(v); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
type errorResponse struct {
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func writeError(w http.ResponseWriter, status int, msg string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(errorResponse{Error: msg})
|
|
}
|