jcoffey-dev
f6c228b87c
The estate could show an operator their infrastructure and had nothing to say to the business paying for it. Two storefronts and the gateway behind both fix that: Magento on two hosts, WooCommerce on one, and pay-01 carrying authorisations with amount, gateway and decline reason. Orders, revenue, average order value, where checkout loses people and why a card was refused now come out of the same log lines the operators are already reading, which is the argument for not running a separate metrics stack beside this one. Two platforms rather than one deliberately. Magento and WooCommerce write about the same events differently, so a panel that groups by service instead of assuming a single shape is the honest way to build one -- and the demo shows that rather than describing it. Order totals are built from a basket of real SKUs at real prices rather than drawn from a distribution, so average order value moves the way one actually moves. Declines rise during the seeded outage window alongside the 5xx rate, because whatever fails requests fails authorisations too. Twenty-seven new alert rules, thresholds calibrated against what the fleet actually emits -- measured on the demo's own week of history rather than guessed. A rule set at the average fires constantly and one set an order of magnitude above it never fires; these sit two to three times the steady-state rate, so they are quiet in normal operation and true during the diurnal peak or the seeded incident. Four are absence rules, because a domain controller or a storefront going silent is not a threshold question. Six new dashboards: fleet health, golden signals, security posture, capacity and storage, commerce, payments. Three limits of the query language found the hard way and worth writing down, because each was discovered by a panel failing rather than by reading: `dc()` does not exist -- the functions are count, sum, avg, min and max; `or` is not supported between structured filters, so a panel spanning tiers filters on the attribute they share and groups by service; and dashboards refuse raw SQL outright. The validator run over all 169 panels and 38 rules now checks every one of those, plus stages, viz types and comparators.
134 lines
3.5 KiB
JSON
134 lines
3.5 KiB
JSON
{
|
|
"name": "Golden signals",
|
|
"description": "Traffic, errors, latency and saturation across everything that serves a request",
|
|
"default_earliest": "-6h",
|
|
"default_latest": "now",
|
|
"panels": [
|
|
{
|
|
"title": "Requests served",
|
|
"query": "| where status>=100 | stats count",
|
|
"viz_type": "single_stat",
|
|
"position_x": 0,
|
|
"position_y": 0,
|
|
"width": 3,
|
|
"height": 3,
|
|
"query_language": "spl",
|
|
"sort_order": 0
|
|
},
|
|
{
|
|
"title": "5xx responses",
|
|
"query": "| where status>=500 | stats count",
|
|
"viz_type": "single_stat",
|
|
"position_x": 3,
|
|
"position_y": 0,
|
|
"width": 3,
|
|
"height": 3,
|
|
"query_language": "spl",
|
|
"sort_order": 1
|
|
},
|
|
{
|
|
"title": "API latency (avg ms)",
|
|
"query": "service=api | stats avg(duration_ms) as avg_ms",
|
|
"viz_type": "single_stat",
|
|
"position_x": 6,
|
|
"position_y": 0,
|
|
"width": 3,
|
|
"height": 3,
|
|
"query_language": "spl",
|
|
"sort_order": 2
|
|
},
|
|
{
|
|
"title": "Slow Postgres statements",
|
|
"query": "service=postgres | where duration_ms>1000 | stats count",
|
|
"viz_type": "single_stat",
|
|
"position_x": 9,
|
|
"position_y": 0,
|
|
"width": 3,
|
|
"height": 3,
|
|
"query_language": "spl",
|
|
"sort_order": 3
|
|
},
|
|
{
|
|
"title": "Traffic by service and host",
|
|
"query": "| where status>=100 | stats count by service, host",
|
|
"viz_type": "bar",
|
|
"position_x": 0,
|
|
"position_y": 3,
|
|
"width": 12,
|
|
"height": 5,
|
|
"viz_config": {
|
|
"x_column": "service",
|
|
"value_column": "count",
|
|
"series_column": "host",
|
|
"stacked": "true"
|
|
},
|
|
"query_language": "spl",
|
|
"sort_order": 4
|
|
},
|
|
{
|
|
"title": "Where the errors are",
|
|
"query": "severity=ERROR | stats count by service, host | sort -count | head 20",
|
|
"viz_type": "table",
|
|
"position_x": 0,
|
|
"position_y": 8,
|
|
"width": 6,
|
|
"height": 5,
|
|
"query_language": "spl",
|
|
"sort_order": 5
|
|
},
|
|
{
|
|
"title": "API latency by route",
|
|
"query": "service=api | stats avg(duration_ms) as avg_ms by route | sort -avg_ms | head 12",
|
|
"viz_type": "bar",
|
|
"position_x": 6,
|
|
"position_y": 8,
|
|
"width": 6,
|
|
"height": 5,
|
|
"viz_config": {
|
|
"x_column": "route",
|
|
"value_column": "avg_ms"
|
|
},
|
|
"query_language": "spl",
|
|
"sort_order": 6
|
|
},
|
|
{
|
|
"title": "Status codes by service",
|
|
"query": "| where status>=100 | stats count by service, status",
|
|
"viz_type": "heatmap",
|
|
"position_x": 0,
|
|
"position_y": 13,
|
|
"width": 12,
|
|
"height": 5,
|
|
"viz_config": {
|
|
"x_column": "service",
|
|
"y_column": "status",
|
|
"value_column": "count"
|
|
},
|
|
"query_language": "spl",
|
|
"sort_order": 7
|
|
},
|
|
{
|
|
"title": "Slow operations, wherever they are",
|
|
"query": "| where duration_ms>1000 | stats count by service, host | sort -count | head 15",
|
|
"viz_type": "table",
|
|
"position_x": 0,
|
|
"position_y": 18,
|
|
"width": 6,
|
|
"height": 5,
|
|
"query_language": "spl",
|
|
"sort_order": 8
|
|
},
|
|
{
|
|
"title": "Saturation by host",
|
|
"query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu, max(mem_used_bytes) as peak_mem by host | sort -avg_cpu | head 15",
|
|
"viz_type": "table",
|
|
"position_x": 6,
|
|
"position_y": 18,
|
|
"width": 6,
|
|
"height": 5,
|
|
"query_language": "spl",
|
|
"sort_order": 9
|
|
}
|
|
]
|
|
}
|