diff --git a/hack/README.md b/hack/README.md index a9c89c1..cc20a84 100644 --- a/hack/README.md +++ b/hack/README.md @@ -19,3 +19,12 @@ monorepos (Kubernetes among them). the pipeline handles Windows-shaped data; doesn't test the real `EvtSubscribe`/ETW integration, which needs actual Windows. See `/docs/phase-1-runbook.md`. +- `demo-simulator/` — the public demo's synthetic world: a fictional + fleet whose agents check in, report CPU/memory/disk, and ship + realistically shaped logs for eight services. Backfills a window of + history, then keeps generating in real time. Distinct from + `benchmark-fixture/` (volume, for the Phase 2 latency benchmark) and + `windows-fixture/` (correctness, for the Windows ingest path). +- `demo-seed/` — the rest of the demo deployment: its reset script, + dashboards, alert rules, and the systemd unit that runs + `demo-simulator`. diff --git a/hack/demo-seed/README.md b/hack/demo-seed/README.md new file mode 100644 index 0000000..ebd9be6 --- /dev/null +++ b/hack/demo-seed/README.md @@ -0,0 +1,79 @@ +# demo-seed + +Everything the public demo deployment (`demo.cairnobs.org`) is built out +of, kept in the repo rather than only on the box so a demo can be rebuilt +from scratch and so its dashboards and rules are reviewable like any +other code. + +- `reset-demo.sh` — the nightly reset. Wipes every volume, brings the + stack back up, re-seeds users, notification targets, a week of + synthetic history, dashboards, and alert rules. Run from cron at 04:00 + on the demo host. +- `dashboards/*.json` — one file per dashboard, in the shape + `POST /dashboards/import` consumes (identical to what + `GET /dashboards/{id}/export` and the web UI's Export JSON button + produce, so a dashboard edited in the UI can be exported straight back + into this directory). +- `alerts/*.json.template` — one file per alert rule, in the shape + `POST /rules` consumes. `__TARGET_OPS__` / `__TARGET_SECURITY__` / + `__TARGET_PLATFORM__` are substituted at apply time with the IDs of the + three notification targets `reset-demo.sh` creates: every reset starts + from an empty database, so the IDs can't be baked in. +- `cairnobs-demo-simulator.service` — systemd unit for the live half of + the demo, `/hack/demo-simulator`. Installed at `/etc/systemd/system/` + on the demo box. + +## Prefilled login + +The demo's login page comes up with the read-only `demo` account already +in both fields, so a visitor doesn't need credentials handed to them. +That's a build-time opt-in, off everywhere else: the web image is built +with `VITE_DEMO_USERNAME`/`VITE_DEMO_PASSWORD` (set in the demo host's +`docker-compose.override.yml`), and the login page prefills only when it +has both. Any deployment that doesn't set them gets the ordinary empty +form -- see `web/src/lib/api.ts`'s `demoUsername`. + +The password is baked into the static bundle, which is fine for exactly +this case and nothing else: a Viewer-role account on a deployment whose +database is wiped and reseeded nightly. It has to match `DEMO_PASSWORD` +in `reset-demo.sh`, and changing that means rebuilding the web image. + +## Why the demo needs a long-running process + +Three things the demo has to show are only true if data keeps arriving, +and no amount of one-shot seeding fixes any of them: + +- **Agents.** The Agents page is populated by the `AgentControl.CheckIn` + RPC, and marks a host stale once it stops calling in. A fleet seeded + once at 04:00 is entirely stale by 04:10. +- **Alerts.** Rules evaluate over trailing windows (`earliest=-5m`). + Against a frozen dataset every rule settles into a permanent state + within minutes and the Alerts page never moves again. +- **Recent views.** A "last 15 minutes" dashboard, or a query for what + just happened, is empty on a dataset that stopped growing overnight. + +So the demo runs `demo-simulator` continuously, and `reset-demo.sh` only +handles the parts that genuinely are one-shot: the history behind the +present, and the dashboards and rules themselves. + +## Editing a dashboard + +Change it in the web UI, hit Export JSON, and drop the file in +`dashboards/` — the export shape and the import shape are the same one. +The next reset picks it up. (Note that panel IDs and the dashboard ID are +not part of that shape: every reset creates them fresh.) + +## What the queries can't do yet + +Every panel here uses `table`, `bar`, `top_n`, `single_stat`, or +`heatmap`. None uses `line`, because a line chart needs a time axis and +the query language has no time-bucketing function — `stats count by +` groups by literal column values, so there's no equivalent of +Splunk's `bin`/`timechart` to group by hour or minute. Raw SQL could +express it (`toStartOfHour(timestamp)`), but dashboard panels reject the +SQL escape hatch by design, since the time-range picker works by +prepending `earliest=`/`latest=` terms to a pipe-syntax query +(see `api/dashboards/types.go`'s `validatePanel`). + +That gap is the one real thing standing between these dashboards and a +conventional observability overview screen. diff --git a/hack/demo-seed/alerts/agent-legacy-01-unavailable.json.template b/hack/demo-seed/alerts/agent-legacy-01-unavailable.json.template new file mode 100644 index 0000000..8bd5aca --- /dev/null +++ b/hack/demo-seed/alerts/agent-legacy-01-unavailable.json.template @@ -0,0 +1,11 @@ +{ + "name": "legacy-01 agent unavailable", + "description": "No heartbeat from legacy-01 within its heartbeat window -- the agent or the host is gone", + "query": "earliest=-5m host=\"legacy-01\" cairnobs.heartbeat=true", + "query_language": "spl", + "condition_type": "absence", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true +} diff --git a/hack/demo-seed/alerts/api-5xx-rate.json.template b/hack/demo-seed/alerts/api-5xx-rate.json.template new file mode 100644 index 0000000..faaf401 --- /dev/null +++ b/hack/demo-seed/alerts/api-5xx-rate.json.template @@ -0,0 +1,13 @@ +{ + "name": "High API 5xx rate", + "description": "The API tier is returning server errors well above its normal background rate", + "query": "service=api earliest=-5m | where status>=500 | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 1, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 4 +} diff --git a/hack/demo-seed/alerts/api-latency.json.template b/hack/demo-seed/alerts/api-latency.json.template new file mode 100644 index 0000000..f75a717 --- /dev/null +++ b/hack/demo-seed/alerts/api-latency.json.template @@ -0,0 +1,13 @@ +{ + "name": "API latency degraded", + "description": "Average API response time over the last 10 minutes is above the service objective", + "query": "service=api earliest=-10m | stats avg(latency_ms) as avg_latency_ms", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 2, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 130 +} diff --git a/hack/demo-seed/alerts/edge-5xx-surge.json.template b/hack/demo-seed/alerts/edge-5xx-surge.json.template new file mode 100644 index 0000000..2cfd78b --- /dev/null +++ b/hack/demo-seed/alerts/edge-5xx-surge.json.template @@ -0,0 +1,13 @@ +{ + "name": "Edge 5xx surge", + "description": "nginx is serving 5xx to clients -- either upstreams are failing or the edge itself is", + "query": "service=nginx earliest=-5m | where status>=500 | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 1, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 4 +} diff --git a/hack/demo-seed/alerts/firewall-block-surge.json.template b/hack/demo-seed/alerts/firewall-block-surge.json.template new file mode 100644 index 0000000..5fb9d61 --- /dev/null +++ b/hack/demo-seed/alerts/firewall-block-surge.json.template @@ -0,0 +1,13 @@ +{ + "name": "Firewall block surge", + "description": "UFW is dropping far more inbound connections than usual -- typically a scan in progress", + "query": "service=system ufw_action=BLOCK earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 120, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 20 +} diff --git a/hack/demo-seed/alerts/mail-auth-failures.json.template b/hack/demo-seed/alerts/mail-auth-failures.json.template new file mode 100644 index 0000000..7115794 --- /dev/null +++ b/hack/demo-seed/alerts/mail-auth-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Mail authentication failures", + "description": "Repeated SMTP auth failures on the mail host -- credential stuffing against the mail server", + "query": "service=smtp result=auth_failed earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 120, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 6 +} diff --git a/hack/demo-seed/alerts/postgres-slow-queries.json.template b/hack/demo-seed/alerts/postgres-slow-queries.json.template new file mode 100644 index 0000000..d4f7209 --- /dev/null +++ b/hack/demo-seed/alerts/postgres-slow-queries.json.template @@ -0,0 +1,13 @@ +{ + "name": "Slow database queries", + "description": "Statements taking over a second are piling up, which usually shows up as API latency next", + "query": "service=postgres earliest=-10m | where duration_ms>1000 | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 2, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 5 +} diff --git a/hack/demo-seed/alerts/ssh-brute-force.json.template b/hack/demo-seed/alerts/ssh-brute-force.json.template new file mode 100644 index 0000000..f08affb --- /dev/null +++ b/hack/demo-seed/alerts/ssh-brute-force.json.template @@ -0,0 +1,13 @@ +{ + "name": "SSH brute-force attempt", + "description": "A burst of failed SSH authentications across the fleet, well past normal background probing", + "query": "service=system auth_result=failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/windows-account-lockout.json.template b/hack/demo-seed/alerts/windows-account-lockout.json.template new file mode 100644 index 0000000..de0b5ba --- /dev/null +++ b/hack/demo-seed/alerts/windows-account-lockout.json.template @@ -0,0 +1,13 @@ +{ + "name": "Windows account lockout", + "description": "Any 4740 on the Windows hosts: an account was locked out after repeated failures", + "query": "winevt.event_id=4740 earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 120, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gte", + "threshold_value": 1 +} diff --git a/hack/demo-seed/alerts/worker-disk-filling.json.template b/hack/demo-seed/alerts/worker-disk-filling.json.template new file mode 100644 index 0000000..555361f --- /dev/null +++ b/hack/demo-seed/alerts/worker-disk-filling.json.template @@ -0,0 +1,13 @@ +{ + "name": "worker-02 disk nearly full", + "description": "worker-02's data volume has passed 175 GiB of 200 GiB and is still climbing", + "query": "host=\"worker-02\" cairnobs.metrics=true earliest=-15m | stats max(disk_used_bytes) as disk_used_bytes", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 187904819200 +} diff --git a/hack/demo-seed/alerts/worker-job-failures.json.template b/hack/demo-seed/alerts/worker-job-failures.json.template new file mode 100644 index 0000000..d95d598 --- /dev/null +++ b/hack/demo-seed/alerts/worker-job-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Background job failures", + "description": "Worker jobs are failing after their retries are exhausted", + "query": "service=worker result=failed earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 120, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/cairnobs-demo-simulator.service b/hack/demo-seed/cairnobs-demo-simulator.service new file mode 100644 index 0000000..f021ace --- /dev/null +++ b/hack/demo-seed/cairnobs-demo-simulator.service @@ -0,0 +1,39 @@ +# The live half of the demo deployment: /hack/demo-simulator kept running +# so the demo's agents keep checking in, its hosts keep reporting metrics, +# and its logs keep arriving. -backfill 0 because history is seeded once +# per night by reset-demo.sh, which stops this unit for the duration and +# starts it again afterwards. +# +# Installed on the demo box at /etc/systemd/system/, not built into any +# image -- this is demo scaffolding, not part of the product. +[Unit] +Description=Cairn OBS demo data simulator +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +User=john +WorkingDirectory=/home/john/cairnobs-demo +ExecStart=/home/john/cairnobs-demo/bin/demo-simulator \ + -addr 127.0.0.1:4317 \ + -ca /home/john/cairnobs-demo/hack/dev-certs/out/ca.pem \ + -cert /home/john/cairnobs-demo/hack/dev-certs/out/client.pem \ + -key /home/john/cairnobs-demo/hack/dev-certs/out/client-key.pem \ + -backfill 0 -live +Restart=always +RestartSec=10 + +# Same unprivileged posture the real agents' units were moved to during +# the 2026-08-19 security remediation -- a data generator has no business +# with write access to anything on this box. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictSUIDSGID=true + +[Install] +WantedBy=multi-user.target diff --git a/hack/demo-seed/dashboards/database-cache.json b/hack/demo-seed/dashboards/database-cache.json new file mode 100644 index 0000000..ae8f02c --- /dev/null +++ b/hack/demo-seed/dashboards/database-cache.json @@ -0,0 +1,113 @@ +{ + "name": "Database & cache", + "description": "Postgres statement latency and Redis memory pressure", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Slow queries (>1s)", + "query": "service=postgres | where duration_ms>1000 | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Avg statement (ms)", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=postgres | stats avg(duration_ms) as avg_duration_ms", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Redis evictions", + "query": "service=redis op=evict | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Statements by kind", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=postgres | stats count by query_kind | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "query_kind", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Slowest tables (avg ms)", + "query": "service=postgres | stats avg(duration_ms) as avg_duration_ms by table | sort -avg_duration_ms | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "table", + "value_column": "avg_duration_ms" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Statement kind by table", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=postgres | stats count by table, query_kind", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "table", + "y_column": "query_kind", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Redis events by operation", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=redis | stats count by op | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "op", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Slowest statements", + "query": "service=postgres | where duration_ms>1000 | sort -duration_ms | head 50 | fields timestamp, table, query_kind, duration_ms, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/errors-reliability.json b/hack/demo-seed/dashboards/errors-reliability.json new file mode 100644 index 0000000..189278c --- /dev/null +++ b/hack/demo-seed/dashboards/errors-reliability.json @@ -0,0 +1,113 @@ +{ + "name": "Errors & reliability", + "description": "Where failures are concentrated right now -- 5xx, fatal events, and failed background jobs", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Error events", + "query": "severity=ERROR | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Fatal events", + "query": "severity=FATAL | stats count", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Failed jobs", + "query": "service=worker result=failed | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Errors by service", + "query": "severity=ERROR | stats count by service | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "service", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Top API error codes", + "query": "service=api | where status>=500 | stats count by error_code | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "error_code", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "5xx by host and status", + "query": "status>=500 | stats count by host, status", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "y_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Failed jobs by queue", + "query": "service=worker result=failed | stats count by queue | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "queue", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Recent 5xx responses", + "query": "status>=500 | sort -timestamp | head 50 | fields timestamp, host, service, status, route, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/infrastructure.json b/hack/demo-seed/dashboards/infrastructure.json new file mode 100644 index 0000000..a42b12a --- /dev/null +++ b/hack/demo-seed/dashboards/infrastructure.json @@ -0,0 +1,112 @@ +{ + "name": "Infrastructure", + "description": "Agent-reported CPU, memory, and disk for every host in the fleet", + "default_earliest": "-6h", + "default_latest": "now", + "panels": [ + { + "title": "Metric samples", + "query": "cairnobs.metrics=true | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Peak CPU (%)", + "query": "cairnobs.metrics=true | stats max(cpu_percent) as max_cpu_percent", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Agent heartbeats", + "query": "cairnobs.heartbeat=true | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Average CPU by host", + "query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu_percent by host | sort -avg_cpu_percent", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "value_column": "avg_cpu_percent" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Disk used by host (bytes)", + "query": "cairnobs.metrics=true | stats max(disk_used_bytes) as disk_used_bytes by host | sort -disk_used_bytes | head 12", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "host", + "value_column": "disk_used_bytes" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Memory used by host (bytes)", + "query": "cairnobs.metrics=true | stats max(mem_used_bytes) as mem_used_bytes by host | sort -mem_used_bytes", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "value_column": "mem_used_bytes" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Heartbeats by host", + "query": "cairnobs.heartbeat=true | stats count by host | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Latest sample per host", + "query": "cairnobs.metrics=true | stats max(uptime_seconds) as uptime_seconds, avg(cpu_percent) as avg_cpu_percent, max(cpu_cores) as cores by host | sort -avg_cpu_percent", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/mail-delivery.json b/hack/demo-seed/dashboards/mail-delivery.json new file mode 100644 index 0000000..d9eb9df --- /dev/null +++ b/hack/demo-seed/dashboards/mail-delivery.json @@ -0,0 +1,113 @@ +{ + "name": "Mail delivery", + "description": "SMTP delivery outcomes, authentication failures, and spam rejections on the mail host", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Delivered", + "query": "service=smtp result=delivered | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Auth failures", + "query": "service=smtp result=auth_failed | stats count", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Spam rejections", + "query": "service=smtp result=spam_reject | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Outcomes", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=smtp | stats count by result | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "result", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Top sender domains", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=smtp | stats count by sender_domain | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "sender_domain", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Outcome by sender domain", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=smtp | stats count by sender_domain, result", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "sender_domain", + "y_column": "result", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Auth failures by source address", + "query": "service=smtp result=auth_failed | stats count by remote_addr | sort -count | head 10", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "remote_addr", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Recent rejections", + "query": "service=smtp result=spam_reject | sort -timestamp | head 50 | fields timestamp, remote_addr, sender_domain, spam_score, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/security.json b/hack/demo-seed/dashboards/security.json new file mode 100644 index 0000000..aab498d --- /dev/null +++ b/hack/demo-seed/dashboards/security.json @@ -0,0 +1,113 @@ +{ + "name": "Security", + "description": "SSH authentication, firewall blocks, and Windows logon failures across the fleet", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Failed SSH logins", + "query": "service=system auth_result=failed | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Firewall blocks", + "query": "service=system ufw_action=BLOCK | stats count", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Windows logon failures", + "query": "winevt.event_id=4625 | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Top source addresses", + "query": "service=system auth_result=failed | stats count by remote_addr | sort -count | head 10", + "viz_type": "top_n", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "remote_addr", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Blocked destination ports", + "query": "service=system ufw_action=BLOCK | stats count by dst_port | sort -count | head 12", + "viz_type": "bar", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "dst_port", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Source address by blocked port", + "query": "service=system ufw_action=BLOCK | stats count by remote_addr, dst_port", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "remote_addr", + "y_column": "dst_port", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Failed logins by host", + "query": "service=system auth_result=failed | stats count by host | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Recent authentication failures", + "query": "service=system auth_result=failed | sort -timestamp | head 50 | fields timestamp, host, ssh_user, remote_addr, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/service-overview.json b/hack/demo-seed/dashboards/service-overview.json new file mode 100644 index 0000000..1c10a4d --- /dev/null +++ b/hack/demo-seed/dashboards/service-overview.json @@ -0,0 +1,113 @@ +{ + "name": "Service overview", + "description": "Every service at a glance: volume, error mix, and the hosts carrying the load", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Log events", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Errors", + "query": "severity=ERROR | stats count", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "5xx responses", + "query": "status>=500 | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Events by service", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by service | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "service", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Busiest hosts", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by host | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Severity by service", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by service, severity", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "service", + "y_column": "severity", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Errors by host", + "query": "severity=ERROR | stats count by host | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Most recent errors", + "query": "severity=ERROR | sort -timestamp | head 50 | fields timestamp, host, service, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/web-api-traffic.json b/hack/demo-seed/dashboards/web-api-traffic.json new file mode 100644 index 0000000..2b3bd7a --- /dev/null +++ b/hack/demo-seed/dashboards/web-api-traffic.json @@ -0,0 +1,113 @@ +{ + "name": "Web & API traffic", + "description": "Edge (nginx) and application (api) tiers: status mix, hot routes, and latency", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Edge requests", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=nginx | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Avg API latency (ms)", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=api | stats avg(latency_ms) as avg_latency_ms", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "API 5xx", + "query": "service=api | where status>=500 | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Edge responses by status", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=nginx | stats count by status | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Busiest routes", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=nginx | stats count by route | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "route", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Status by edge host", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=nginx | stats count by host, status", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "host", + "y_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Slowest API routes (avg ms)", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=api | stats avg(latency_ms) as avg_latency_ms by route | sort -avg_latency_ms | head 10", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "route", + "value_column": "avg_latency_ms" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Slow API requests (>800ms)", + "query": "service=api | where latency_ms>800 | sort -timestamp | head 50 | fields timestamp, host, route, status, latency_ms, trace_id", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/windows-events.json b/hack/demo-seed/dashboards/windows-events.json new file mode 100644 index 0000000..ab525d0 --- /dev/null +++ b/hack/demo-seed/dashboards/windows-events.json @@ -0,0 +1,113 @@ +{ + "name": "Windows events", + "description": "Security, System, and Application channels from the Windows hosts", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Windows events", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=eventlog | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Failed logons (4625)", + "query": "winevt.event_id=4625 | stats count", + "viz_type": "single_stat", + "position_x": 4, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Account lockouts (4740)", + "query": "winevt.event_id=4740 | stats count", + "viz_type": "single_stat", + "position_x": 8, + "position_y": 0, + "width": 4, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Events by ID", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=eventlog | stats count by winevt.event_id | sort -count | head 12", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "winevt.event_id", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Top providers", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=eventlog | stats count by winevt.provider | sort -count | head 10", + "viz_type": "top_n", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "winevt.provider", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Channel by computer", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=eventlog | stats count by winevt.computer, winevt.channel", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "winevt.computer", + "y_column": "winevt.channel", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Severity mix", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true service=eventlog | stats count by severity | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "severity", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Recent security-channel events", + "query": "service=eventlog winevt.channel=Security | sort -timestamp | head 50 | fields timestamp, winevt.computer, winevt.event_id, winevt.target_user, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 6, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/reset-demo.sh b/hack/demo-seed/reset-demo.sh new file mode 100755 index 0000000..a3e89f0 --- /dev/null +++ b/hack/demo-seed/reset-demo.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Nightly reset for the demo.cairnobs.org stack: wipes every data volume +# and re-seeds from scratch, so the demo's timestamps stay recent, its +# incidents stay at the same recent offsets, and storage doesn't grow +# forever. Mirrors the teardown docs/phase-0-runbook.md and +# docs/phase-2-runbook.md already document (`docker compose down -v`), +# plus the seed sequence below. +# +# Seeding is data-first, config-second: dashboards and alert rules are +# applied after the backfill exists, so nothing renders an empty panel or +# evaluates against an empty table on its first pass. +# +# The live half of the demo (/hack/demo-simulator running as +# cairnobs-demo-simulator.service) is stopped for the duration and +# started again at the end -- it must not be pushing records into a stack +# that's being torn down, and it must re-register its agents against the +# fresh, empty `agents` table afterwards. +set -euo pipefail + +DEMO_ROOT=${DEMO_ROOT:-/home/john/cairnobs-demo} +SEED_DIR="$DEMO_ROOT/hack/demo-seed" +CERTS="$DEMO_ROOT/hack/dev-certs/out" +BACKFILL=${BACKFILL:-168h} +RATE_SCALE=${RATE_SCALE:-0.5} +SIMULATOR_UNIT=cairnobs-demo-simulator.service + +cd "$DEMO_ROOT" + +ADMIN_PASSWORD_FILE="$DEMO_ROOT/.admin-password" +# Changing DEMO_PASSWORD means rebuilding the web image too: the login +# page prefills this account's credentials, and they're baked into the +# bundle at build time from VITE_DEMO_USERNAME/VITE_DEMO_PASSWORD in the +# demo host's docker-compose.override.yml. Change one without the other +# and the demo's own login form stops working. +DEMO_PASSWORD='CairnDemo_2026!' +EVALUATOR_PASSWORD='REDACTED_ROTATED_CREDENTIAL' + +echo "=== $(date -u +%FT%TZ) reset starting ===" + +sudo systemctl stop "$SIMULATOR_UNIT" || true + +docker compose down -v +docker compose up -d +echo "waiting for api to report healthy..." +until [ "$(docker inspect -f '{{.State.Health.Status}}' cairnobs-api 2>/dev/null)" = "healthy" ]; do sleep 2; done + +# -seed-admin is idempotent and prints the password once -- capture it +# fresh each run rather than reusing a stale one from a prior reset. +ADMIN_PASSWORD=$(docker compose run --rm api -seed-admin 2>&1 | grep '^ password:' | awk '{print $2}') +echo "$ADMIN_PASSWORD" > "$ADMIN_PASSWORD_FILE" +chmod 600 "$ADMIN_PASSWORD_FILE" + +export CAIRNOBSCTL_API_URL=http://localhost:8080 +export CAIRNOBSCTL_ALERTING_API_URL=http://localhost:8081 +ADMIN_TOKEN=$(echo "$ADMIN_PASSWORD" | ./bin/cairnobsctl users login admin) +export CAIRNOBSCTL_TOKEN="$ADMIN_TOKEN" + +# The password reaches the CLI on stdin only -- `--password ` was +# removed deliberately (see cli/cmd/cairnobsctl/cmd_users.go). +echo "$DEMO_PASSWORD" | ./bin/cairnobsctl users create demo --role viewer >/dev/null +echo "$EVALUATOR_PASSWORD" | ./bin/cairnobsctl users create alerting-evaluator --role viewer >/dev/null + +SVCTOKEN=$(echo "$EVALUATOR_PASSWORD" | ./bin/cairnobsctl users login alerting-evaluator) +printf 'COMPOSE_PROFILES=single-tenant\nALERTING_SERVICE_TOKEN=%s\n' "$SVCTOKEN" > .env && chmod 600 .env +docker compose up -d alerting + +# Three notification targets so the Alerts page shows rules routed to +# different destinations, the way a real deployment splits ops/security/ +# platform. The URLs are deliberately inert placeholders on a domain +# reserved for documentation -- nothing is actually notified. +create_target() { + curl -s -X POST http://localhost:8081/targets \ + -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \ + -d "{\"name\":\"$1\",\"kind\":\"webhook\",\"webhook_url\":\"$2\"}" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' +} +TARGET_OPS=$(create_target 'Ops on-call (placeholder)' 'https://example.com/webhooks/ops-oncall') +TARGET_SECURITY=$(create_target 'Security team (placeholder)' 'https://example.com/webhooks/security') +TARGET_PLATFORM=$(create_target 'Platform team (placeholder)' 'https://example.com/webhooks/platform') + +echo "backfilling $BACKFILL of synthetic history..." +./bin/demo-simulator \ + -addr 127.0.0.1:4317 -ca "$CERTS/ca.pem" -cert "$CERTS/client.pem" -key "$CERTS/client-key.pem" \ + -backfill "$BACKFILL" -rate-scale "$RATE_SCALE" -live=false + +# A handful of Windows-shaped records from the dedicated fixture as well: +# it's the tool the Windows ingest path is actually verified with, so +# keeping its output present means the demo and that check agree. +docker run --rm --network host -v "$DEMO_ROOT":/src -w /src/hack/windows-fixture -e GOCACHE=/tmp/gocache golang:1.25-bookworm \ + go run . --addr 127.0.0.1:4317 --ca /src/hack/dev-certs/out/ca.pem --cert /src/hack/dev-certs/out/client.pem --key /src/hack/dev-certs/out/client-key.pem --count 5 + +echo "applying dashboards..." +for f in "$SEED_DIR"/dashboards/*.json; do + ./bin/cairnobsctl dashboards apply "$f" >/dev/null + echo " $(basename "$f")" +done + +echo "applying alert rules..." +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +for f in "$SEED_DIR"/alerts/*.json.template; do + out="$TMP/$(basename "${f%.template}")" + sed -e "s/__TARGET_OPS__/$TARGET_OPS/" \ + -e "s/__TARGET_SECURITY__/$TARGET_SECURITY/" \ + -e "s/__TARGET_PLATFORM__/$TARGET_PLATFORM/" "$f" > "$out" + ./bin/cairnobsctl alerts apply "$out" >/dev/null + echo " $(basename "$out")" +done + +sudo systemctl start "$SIMULATOR_UNIT" + +echo "=== $(date -u +%FT%TZ) reset complete ===" diff --git a/hack/demo-simulator/checkin.go b/hack/demo-simulator/checkin.go new file mode 100644 index 0000000..4d98d9a --- /dev/null +++ b/hack/demo-simulator/checkin.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "log" + "sync" + "time" + + agentv1 "github.com/cairnobs/cairnobs/proto/sentry/agent/v1" +) + +// The Agents page is populated by the AgentControl.CheckIn RPC, not by +// log volume: ingest's internal/agentregistry upserts one row per +// (tenant, host) on every check-in, and nothing else ever writes that +// table. A demo with plenty of logs but no check-ins therefore shows an +// empty Agents page -- which is exactly the state this replaces. +// +// The simulated agents are faithful to the real protocol in the two +// places a demo viewer can actually observe it: +// +// - Remote config edits round-trip. The response's DesiredOverride +// version is echoed back as applied_override_version on the next +// check-in, so editing an agent's batch size or log paths in the web +// UI shows the real pending -> applied transition instead of a badge +// stuck on "pending" forever. +// - Restart commands are consumed. A queued RESTART is delivered +// at-most-once and cleared by ingest the moment it hands it out; the +// simulated agent acknowledges it by logging and resetting its +// applied-version state, the same visible outcome a real restart has. +type simAgent struct { + h *host + // appliedVersion is what this agent reports having applied. Starts + // empty (never applied an override) and follows whatever the server + // hands back, one check-in behind -- the same one-tick lag a real + // agent has. + appliedVersion string + // applied is the override itself, kept so the *reported* config on + // subsequent check-ins reflects it. A real agent restarts its + // batcher/heartbeat with the new settings and then reports the new + // values back; without this the web UI would show a config marked + // "applied" next to reported values that never changed, which reads + // like the edit silently failed. + applied *agentv1.DesiredOverride +} + +// reportedConfig is what the agent says it is currently running: its +// local agent.toml settings, with any applied override layered on top. +func (a *simAgent) reportedConfig() *agentv1.ReportedConfig { + cfg := &agentv1.ReportedConfig{ + AgentVersion: a.h.agentVersion, + SourceKind: a.h.sourceKind, + SourceDetail: a.h.sourceDetail, + BatchMaxSize: uint64(a.h.batchMax), + BatchFlushIntervalMs: uint64(a.h.batchFlushMS), + HeartbeatEnabled: true, + HeartbeatIntervalMs: uint64(a.h.heartbeatMS), + } + if o := a.applied; o != nil { + if o.BatchMaxSize != nil { + cfg.BatchMaxSize = o.GetBatchMaxSize() + } + if o.BatchFlushIntervalMs != nil { + cfg.BatchFlushIntervalMs = o.GetBatchFlushIntervalMs() + } + if o.HeartbeatEnabled != nil { + cfg.HeartbeatEnabled = o.GetHeartbeatEnabled() + } + if o.HeartbeatIntervalMs != nil { + cfg.HeartbeatIntervalMs = o.GetHeartbeatIntervalMs() + } + // A journald-unit override changes what the agent is tailing, + // which is exactly what source_detail describes. + if o.JournaldUnit != nil && a.h.sourceKind == "journald" { + if u := o.GetJournaldUnit(); u != "" { + cfg.SourceDetail = "unit=" + u + } else { + cfg.SourceDetail = "whole journal" + } + } + } + return cfg +} + +func (a *simAgent) checkIn(ctx context.Context, client agentv1.AgentControlClient) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + resp, err := client.CheckIn(ctx, &agentv1.CheckInRequest{ + Host: a.h.name, + Service: a.h.service, + CurrentConfig: a.reportedConfig(), + AppliedOverrideVersion: a.appliedVersion, + }) + if err != nil { + return err + } + + if resp.GetHasOverride() && resp.GetOverride() != nil { + if v := resp.GetOverride().GetVersion(); v != a.appliedVersion { + log.Printf("agent %s: applying remote config override version %s", a.h.name, v) + a.appliedVersion = v + a.applied = resp.GetOverride() + } + } else if a.applied != nil { + // The override was cleared (DELETE /agents/{host}/config). A real + // agent falls back to its local agent.toml at that point, and so + // must this one -- otherwise the web UI shows an agent with no + // override still reporting the settings that override gave it, + // and the Clear button looks broken. + log.Printf("agent %s: remote config override cleared, reverting to local config", a.h.name) + a.applied = nil + a.appliedVersion = "" + } + if resp.GetPendingCommand() == agentv1.AgentCommand_AGENT_COMMAND_RESTART { + log.Printf("agent %s: restart command received, simulating restart", a.h.name) + } + return nil +} + +// runCheckIns keeps every non-stale agent checking in on its own +// heartbeat interval for as long as ctx lives. Stale hosts check in once +// at startup and never again, which is what puts a genuine "stale" row +// on the Agents page a few minutes into any demo session. +func runCheckIns(ctx context.Context, client agentv1.AgentControlClient) { + var agents []*simAgent + for i := range fleet { + agents = append(agents, &simAgent{h: &fleet[i]}) + } + + var wg sync.WaitGroup + for _, a := range agents { + if err := a.checkIn(ctx, client); err != nil { + log.Printf("agent %s: initial check-in failed: %v", a.h.name, err) + } + if a.h.stale { + continue + } + wg.Add(1) + go func(a *simAgent) { + defer wg.Done() + ticker := time.NewTicker(time.Duration(a.h.heartbeatMS) * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := a.checkIn(ctx, client); err != nil && ctx.Err() == nil { + log.Printf("agent %s: check-in failed: %v", a.h.name, err) + } + } + } + }(a) + } + log.Printf("registered %d simulated agents (%d checking in every heartbeat interval)", len(agents), len(agents)-staleCount()) + wg.Wait() +} + +func staleCount() int { + n := 0 + for i := range fleet { + if fleet[i].stale { + n++ + } + } + return n +} diff --git a/hack/demo-simulator/events.go b/hack/demo-simulator/events.go new file mode 100644 index 0000000..f0bf19f --- /dev/null +++ b/hack/demo-simulator/events.go @@ -0,0 +1,655 @@ +package main + +import ( + "fmt" + "math/rand" + "strconv" + "strings" + "time" + + logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1" +) + +// One generator per service. Each returns a record whose *message* reads +// like the real thing that service writes (combined-log-format nginx +// lines, Postgres duration/statement lines, Stalwart-shaped SMTP events, +// sshd/UFW journald lines) and whose *attributes* carry the structured +// fields those messages contain. Both halves matter: the message is what +// free-text search (`message:"connection refused"`) matches, the +// attributes are what `where status>=500` and `stats ... by path` +// aggregate over, and a demo that only had one of them would leave half +// the query language with nothing to show. + +var ( + // Two IP pools, deliberately distinct: legitimate client traffic in + // documentation ranges, and a small set of "attacker" addresses that + // recur across sshd failures and UFW blocks, so a viewer who spots + // one in the Security dashboard can pivot on it and find the rest. + clientIPs = []string{ + "203.0.113.14", "203.0.113.72", "203.0.113.109", "198.51.100.7", + "198.51.100.42", "192.0.2.28", "192.0.2.155", "203.0.113.201", + } + attackerIPs = []string{ + "45.155.205.233", "185.191.171.12", "89.248.165.74", "141.98.11.60", + } + + userAgents = []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0", + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Mobile/15E148 Safari/604.1", + "curl/8.11.1", + "shop-mobile-android/4.12.0 (okhttp/4.12.0)", + "Googlebot/2.1 (+http://www.google.com/bot.html)", + } + + // Routes are shared between the nginx and api generators on purpose: + // the same request shows up in the edge tier's access log and the + // application tier's own log, which is exactly what makes a + // "requests by path" panel comparable across services. + routes = []struct { + method string + path string + route string + weight int + slow bool + }{ + {"GET", "/", "/", 14, false}, + {"GET", "/products", "/products", 12, false}, + {"GET", "/products/%d", "/products/:id", 16, false}, + {"GET", "/api/v1/cart", "/api/v1/cart", 9, false}, + {"POST", "/api/v1/cart/items", "/api/v1/cart/items", 7, false}, + {"POST", "/api/v1/checkout", "/api/v1/checkout", 5, true}, + {"GET", "/api/v1/orders", "/api/v1/orders", 6, false}, + {"GET", "/api/v1/orders/%d", "/api/v1/orders/:id", 5, false}, + {"POST", "/api/v1/auth/login", "/api/v1/auth/login", 6, false}, + {"GET", "/api/v1/search", "/api/v1/search", 8, true}, + {"GET", "/static/app.%s.js", "/static/*", 10, false}, + {"GET", "/healthz", "/healthz", 4, false}, + {"POST", "/api/v1/webhooks/stripe", "/api/v1/webhooks/stripe", 3, false}, + } + routeWeightTotal int + + regions = []string{"us-east-1", "us-west-2", "eu-west-1"} + appVerson = "shop-api@2026.8.3" +) + +func init() { + for _, r := range routes { + routeWeightTotal += r.weight + } +} + +func pickRoute(r *rand.Rand) (method, path, route string, slow bool) { + n := r.Intn(routeWeightTotal) + for _, rt := range routes { + if n -= rt.weight; n < 0 { + p := rt.path + switch { + case strings.Contains(p, "%d"): + p = fmt.Sprintf(p, 1000+r.Intn(9000)) + case strings.Contains(p, "%s"): + p = fmt.Sprintf(p, hexString(r, 8)) + } + return rt.method, p, rt.route, rt.slow + } + } + return "GET", "/", "/", false +} + +func hexString(r *rand.Rand, n int) string { + const hexDigits = "0123456789abcdef" + b := make([]byte, n) + for i := range b { + b[i] = hexDigits[r.Intn(16)] + } + return string(b) +} + +func pick[T any](r *rand.Rand, xs []T) T { return xs[r.Intn(len(xs))] } + +func newRecord(h *host, service string, t time.Time, sev logsv1.Severity, msg string, attrs map[string]string) *logsv1.LogRecord { + return &logsv1.LogRecord{ + TimestampUnixNano: t.UnixNano(), + Host: h.name, + Service: service, + Severity: sev, + Message: msg, + Attributes: attrs, + } +} + +// severityForStatus keeps the severity column and the status attribute +// telling the same story -- a 500 that logged at INFO would make +// `severity=ERROR` and `where status>=500` disagree, and a demo where +// two obvious queries contradict each other is worse than one with less +// data. +func severityForStatus(status int) logsv1.Severity { + switch { + case status >= 500: + return logsv1.Severity_SEVERITY_ERROR + case status >= 400: + return logsv1.Severity_SEVERITY_WARN + default: + return logsv1.Severity_SEVERITY_INFO + } +} + +func nginxRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + method, path, route, slow := pickRoute(r) + clientIP := pick(r, clientIPs) + + // The edge tier mirrors whatever the API tier is doing: during the + // outage window a good share of upstream requests come back 5xx here + // too. Outside one it still isn't zero -- a real edge always returns + // the occasional 502 from an upstream restarting or a slow request + // tripping proxy_read_timeout, and a demo whose 5xx count is exactly + // zero for six days straight makes every 5xx panel and rule look + // broken rather than healthy. + upstreamErrRate := c.apiErrorRate + if upstreamErrRate == 0 { + upstreamErrRate = 0.008 + } + status := 200 + switch { + case r.Float64() < upstreamErrRate: + status = pick(r, []int{502, 503, 504}) + case r.Float64() < 0.04: + status = pick(r, []int{404, 401, 403, 429}) + case r.Float64() < 0.06: + status = 301 + } + + base := 40 + r.Float64()*180 + if slow { + base *= 3 + } + latency := base * c.latencyMult * (0.6 + r.Float64()*0.9) + bytes := 400 + r.Intn(60000) + referrer := "-" + if r.Float64() < 0.55 { + referrer = "https://shop.example.com" + pick(r, []string{"/", "/products", "/cart"}) + } + ua := pick(r, userAgents) + + msg := fmt.Sprintf(`%s - - [%s] "%s %s HTTP/1.1" %d %d %q %q %.3f`, + clientIP, t.UTC().Format("02/Jan/2006:15:04:05 -0700"), + method, path, status, bytes, referrer, ua, latency/1000) + + attrs := map[string]string{ + "remote_addr": clientIP, + "method": method, + "path": path, + "route": route, + "status": strconv.Itoa(status), + "bytes": strconv.Itoa(bytes), + "latency_ms": fmt.Sprintf("%.1f", latency), + "referrer": referrer, + "user_agent": ua, + "vhost": "shop.example.com", + } + + // A slice of edge-tier traffic is error-log lines rather than + // access-log ones -- the same thing a real nginx host ships from two + // files under one service. + if status >= 500 && r.Float64() < 0.5 { + upstream := fmt.Sprintf("10.0.2.%d:8080", 21+r.Intn(3)) + msg = fmt.Sprintf(`%s [error] %d#0: *%d connect() failed (111: Connection refused) while connecting to upstream, client: %s, server: shop.example.com, request: "%s %s HTTP/1.1", upstream: "http://%s%s"`, + t.UTC().Format("2006/01/02 15:04:05"), 1000+r.Intn(900), r.Intn(90000), clientIP, method, path, upstream, path) + attrs["upstream_addr"] = upstream + attrs["log_kind"] = "error" + } else { + attrs["log_kind"] = "access" + } + + return newRecord(h, "nginx", t, severityForStatus(status), msg, attrs) +} + +var apiErrors = []struct { + code string + detail string +}{ + {"db_pool_exhausted", "could not acquire a database connection: pool exhausted after 5000ms"}, + {"upstream_timeout", "payment provider request timed out after 30s"}, + {"null_reference", "unhandled exception in OrderService.finalize: nil pointer dereference"}, + {"serialization_failure", "could not serialize access due to concurrent update"}, + {"rate_limited_upstream", "inventory service returned 429, giving up after 3 retries"}, +} + +func apiRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + method, path, route, slow := pickRoute(r) + traceID := hexString(r, 16) + userID := fmt.Sprintf("u_%d", 1000+r.Intn(4000)) + region := pick(r, regions) + + errRate := c.apiErrorRate + if errRate == 0 { + errRate = 0.012 // the healthy baseline: a real service is never at exactly zero + } + + status := 200 + switch { + case r.Float64() < errRate: + status = pick(r, []int{500, 503}) + case r.Float64() < 0.05: + status = pick(r, []int{400, 401, 404, 422, 429}) + case method == "POST" && r.Float64() < 0.3: + status = 201 + } + + base := 25 + r.Float64()*120 + if slow { + base *= 3.5 + } + latency := base * c.latencyMult * (0.6 + r.Float64()*0.8) + dbTime := latency * (0.2 + r.Float64()*0.5) + + attrs := map[string]string{ + "method": method, + "path": path, + "route": route, + "status": strconv.Itoa(status), + "latency_ms": fmt.Sprintf("%.1f", latency), + "db_time_ms": fmt.Sprintf("%.1f", dbTime), + "trace_id": traceID, + "user_id": userID, + "region": region, + "version": appVerson, + } + + var msg string + if status >= 500 { + e := pick(r, apiErrors) + attrs["error_code"] = e.code + msg = fmt.Sprintf("%s %s -> %d in %.0fms trace=%s: %s", method, path, status, latency, traceID, e.detail) + } else { + msg = fmt.Sprintf("%s %s -> %d in %.0fms trace=%s user=%s region=%s", method, path, status, latency, traceID, userID, region) + } + return newRecord(h, "api", t, severityForStatus(status), msg, attrs) +} + +var workerJobs = []struct { + name string + queue string + msMin int + msMax int +}{ + {"order.confirmation_email", "email", 120, 900}, + {"report.daily_sales", "reports", 4000, 22000}, + {"inventory.reconcile", "inventory", 800, 6000}, + {"image.thumbnail", "media", 200, 2500}, + {"search.reindex", "search", 3000, 30000}, + {"webhook.retry", "webhooks", 100, 1500}, +} + +func workerRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + j := pick(r, workerJobs) + dur := float64(j.msMin+r.Intn(j.msMax-j.msMin)) * c.latencyMult + queueDepth := r.Intn(40) + attempt := 1 + if r.Float64() < 0.12 { + attempt = 2 + r.Intn(2) + } + + attrs := map[string]string{ + "job": j.name, + "queue": j.queue, + "duration_ms": fmt.Sprintf("%.0f", dur), + "attempt": strconv.Itoa(attempt), + "queue_depth": strconv.Itoa(queueDepth), + } + + failRate := c.jobFailureRate + if failRate == 0 { + failRate = 0.05 // normal operation: retries exist because jobs do fail + } + + switch { + case r.Float64() < failRate: + attrs["result"] = "failed" + reason := pick(r, []string{ + "SMTP connection refused by relay", + "request timeout after 30s calling inventory-service", + "deadlock detected while updating orders", + }) + return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("job %s failed after %d attempts in %.0fms: %s", j.name, attempt, dur, reason), attrs) + case queueDepth > 32: + attrs["result"] = "ok" + return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("job %s completed in %.0fms but queue %s is backing up (depth=%d)", j.name, dur, j.queue, queueDepth), attrs) + default: + attrs["result"] = "ok" + return newRecord(h, "worker", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("job %s completed in %.0fms (queue=%s attempt=%d)", j.name, dur, j.queue, attempt), attrs) + } +} + +var pgStatements = []struct { + kind string + table string + sql string + msMin int + msMax int +}{ + {"SELECT", "orders", "SELECT o.*, c.email FROM orders o JOIN customers c ON c.id = o.customer_id WHERE o.customer_id = $1 ORDER BY o.created_at DESC LIMIT 50", 4, 120}, + {"SELECT", "products", "SELECT * FROM products WHERE tsv @@ plainto_tsquery($1) LIMIT 100", 30, 900}, + {"INSERT", "order_items", "INSERT INTO order_items (order_id, product_id, qty, price_cents) VALUES ($1, $2, $3, $4)", 2, 40}, + {"UPDATE", "inventory", "UPDATE inventory SET on_hand = on_hand - $1 WHERE sku = $2", 3, 250}, + {"SELECT", "sessions", "SELECT * FROM sessions WHERE token = $1", 1, 20}, + {"DELETE", "carts", "DELETE FROM carts WHERE updated_at < now() - interval '30 days'", 200, 4000}, +} + +func postgresRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + // A minority of Postgres log volume is lifecycle/connection noise + // rather than statement logging, same as the real thing. + if r.Float64() < 0.18 { + switch r.Intn(3) { + case 0: + conns := 20 + r.Intn(90) + return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("connection authorized: user=shop_app database=shop application_name=%s SSL enabled", appVerson), + map[string]string{"db": "shop", "db_user": "shop_app", "connections": strconv.Itoa(conns), "query_kind": "connect"}) + case 1: + return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("checkpoint complete: wrote %d buffers (%.1f%%); %d WAL file(s) added; sync=%.3f s, total=%.3f s", + 800+r.Intn(4000), r.Float64()*8, r.Intn(4), r.Float64(), 1+r.Float64()*6), + map[string]string{"db": "shop", "query_kind": "checkpoint"}) + default: + return newRecord(h, "postgres", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("could not receive data from client: Connection reset by peer (pid=%d)", 2000+r.Intn(8000)), + map[string]string{"db": "shop", "query_kind": "connection_error", "pid": strconv.Itoa(2000 + r.Intn(8000))}) + } + } + + s := pick(r, pgStatements) + dur := float64(s.msMin+r.Intn(s.msMax-s.msMin)) * c.latencyMult + attrs := map[string]string{ + "db": "shop", + "db_user": "shop_app", + "query_kind": s.kind, + "table": s.table, + "duration_ms": fmt.Sprintf("%.1f", dur), + "rows": strconv.Itoa(r.Intn(500)), + "pid": strconv.Itoa(2000 + r.Intn(8000)), + } + sev := logsv1.Severity_SEVERITY_DEBUG + if dur > 1000 { + sev = logsv1.Severity_SEVERITY_WARN + } + return newRecord(h, "postgres", t, sev, + fmt.Sprintf("duration: %.3f ms statement: %s", dur, s.sql), attrs) +} + +func redisRecord(h *host, t time.Time, r *rand.Rand, _ conditions) *logsv1.LogRecord { + used := int64(3<<30) + r.Int63n(2<<30) + clients := 40 + r.Intn(160) + attrs := map[string]string{ + "used_memory_bytes": strconv.FormatInt(used, 10), + "connected_clients": strconv.Itoa(clients), + } + switch n := r.Intn(10); { + case n < 4: + attrs["op"] = "bgsave" + return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Background saving terminated with success (%d changes in %d seconds)", 10000+r.Intn(50000), 60), attrs) + case n < 6: + evicted := 200 + r.Intn(4000) + attrs["op"] = "evict" + attrs["keys_evicted"] = strconv.Itoa(evicted) + return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("Evicted %d keys to stay under maxmemory (used_memory=%.1fGB)", evicted, float64(used)/float64(1<<30)), attrs) + case n < 8: + attrs["op"] = "client" + return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Accepted %s:%d (connected_clients=%d)", pick(r, []string{"10.0.2.21", "10.0.2.22", "10.0.2.23", "10.0.3.31"}), 40000+r.Intn(20000), clients), attrs) + case n < 9: + attrs["op"] = "replication" + return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_INFO, + "Synchronization with replica 10.0.4.52:6379 succeeded", attrs) + default: + attrs["op"] = "slowlog" + micros := 12000 + r.Intn(90000) + attrs["latency_ms"] = fmt.Sprintf("%.1f", float64(micros)/1000) + return newRecord(h, "redis", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("slowlog entry: KEYS session:* took %d usec", micros), attrs) + } +} + +var mailDomains = []string{"example.com", "example.net", "mail.example.org", "shop.example.com", "gmail.com", "outlook.com"} + +func smtpRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + queueID := hexString(r, 12) + remote := pick(r, clientIPs) + if c.spamWave || r.Float64() < 0.15 { + remote = pick(r, attackerIPs) + } + sender := pick(r, mailDomains) + rcpt := pick(r, []string{"shop.example.com", "cairnobs.example.com"}) + size := 2000 + r.Intn(400000) + + authFailRate := 0.06 + spamRate := 0.08 + if c.spamWave { + authFailRate = 0.45 + spamRate = 0.35 + } + + attrs := map[string]string{ + "queue_id": queueID, + "remote_addr": remote, + "sender_domain": sender, + "rcpt_domain": rcpt, + "size_bytes": strconv.Itoa(size), + } + + switch { + case r.Float64() < authFailRate: + user := pick(r, []string{"admin", "postmaster", "info", "sales", "test"}) + attrs["result"] = "auth_failed" + attrs["auth_user"] = user + return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("auth-not-allowed: authentication failed for user %q from [%s] (mechanism=PLAIN)", user, remote), attrs) + case r.Float64() < spamRate: + score := 6 + r.Float64()*12 + attrs["result"] = "spam_reject" + attrs["spam_score"] = fmt.Sprintf("%.1f", score) + return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("spam-reject: message <%s@%s> from [%s] rejected, score %.1f above threshold 5.0", queueID, sender, remote, score), attrs) + case r.Float64() < 0.1: + attrs["result"] = "deferred" + return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("deferred: <%s> to @%s temporarily rejected (450 4.2.1 mailbox busy), retry in 15m", queueID, rcpt), attrs) + default: + attrs["result"] = "delivered" + return newRecord(h, "smtp", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("delivered: <%s> from @%s to @%s size=%d in %.2fs", queueID, sender, rcpt, size, 0.2+r.Float64()*3), attrs) + } +} + +// internetFacing reports whether a host takes connections straight off +// the internet. Only these see a probe window: an internal host behind +// the edge tier keeps its ordinary background noise either way, and +// pretending otherwise would show a brute-force burst arriving +// simultaneously on hosts that aren't reachable at all. +func internetFacing(h *host) bool { return internetFacingName(h.name) } + +func internetFacingName(name string) bool { + return name == "mail-01" || strings.HasPrefix(name, "edge-") +} + +// systemRecord is the journald stream every Linux host ships: sshd, UFW, +// systemd units, and the occasional kernel message. This is the stream +// the Security dashboard and the brute-force/UFW alert rules read. +func systemRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + pid := strconv.Itoa(400 + r.Intn(30000)) + + // During a probe window an internet-facing host's system stream is + // dominated by failed logins and firewall blocks. + probing := c.bruteForce && internetFacing(h) + roll := r.Float64() + if probing { + roll *= 0.35 + } + + switch { + case roll < 0.22: + src := pick(r, attackerIPs) + user := pick(r, []string{"admin", "root", "ubuntu", "oracle", "postgres", "git", "test"}) + port := 40000 + r.Intn(20000) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("sshd[%s]: Failed password for invalid user %s from %s port %d ssh2", pid, user, src, port), + map[string]string{ + "unit": "ssh.service", "pid": pid, "remote_addr": src, + "ssh_user": user, "src_port": strconv.Itoa(port), "auth_result": "failed", + }) + case roll < 0.4: + src := pick(r, attackerIPs) + dport := pick(r, []int{22, 23, 445, 3389, 5432, 6379, 8080, 3306}) + sport := 40000 + r.Intn(20000) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("kernel: [UFW BLOCK] IN=eth0 OUT= MAC=00:16:3e:%s SRC=%s DST=%s LEN=60 TOS=0x00 PREC=0x00 TTL=52 ID=%d PROTO=TCP SPT=%d DPT=%d WINDOW=1024 SYN", + hexString(r, 2)+":"+hexString(r, 2)+":"+hexString(r, 2), src, h.ipv4, r.Intn(65000), sport, dport), + map[string]string{ + "unit": "kernel", "remote_addr": src, "ufw_action": "BLOCK", + "dst_port": strconv.Itoa(dport), "src_port": strconv.Itoa(sport), "proto": "TCP", + }) + case roll < 0.52: + user := pick(r, []string{"john", "deploy", "ansible"}) + src := pick(r, []string{"203.0.113.5", "10.0.0.9", "198.51.100.7"}) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("sshd[%s]: Accepted publickey for %s from %s port %d ssh2: ED25519 SHA256:%s", pid, user, src, 40000+r.Intn(20000), hexString(r, 20)), + map[string]string{ + "unit": "ssh.service", "pid": pid, "remote_addr": src, + "ssh_user": user, "auth_result": "accepted", + }) + case roll < 0.62: + user := pick(r, []string{"john", "deploy"}) + cmd := pick(r, []string{"/usr/bin/systemctl restart shop-api.service", "/usr/bin/apt-get update", "/usr/bin/journalctl -u shop-worker -n 200"}) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("sudo: %s : TTY=pts/0 ; PWD=/home/%s ; USER=root ; COMMAND=%s", user, user, cmd), + map[string]string{"unit": "sudo", "ssh_user": user, "command": cmd}) + case roll < 0.8: + unit := pick(r, []string{"logrotate.service", "apt-daily.service", "systemd-tmpfiles-clean.service", "fstrim.service"}) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("systemd[1]: %s: Deactivated successfully.", unit), + map[string]string{"unit": unit, "pid": "1"}) + case roll < 0.92: + unit := pick(r, []string{"shop-api.service", "shop-worker.service", "nginx.service", "cairnobs-agent.service"}) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("systemd[1]: Reloaded %s.", unit), + map[string]string{"unit": unit, "pid": "1"}) + case roll < 0.97: + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("kernel: TCP: request_sock_TCP: Possible SYN flooding on port 443. Sending cookies. Check SNMP counters."), + map[string]string{"unit": "kernel", "dst_port": "443"}) + default: + proc := pick(r, []string{"python3", "node", "ruby"}) + return newRecord(h, "system", t, logsv1.Severity_SEVERITY_FATAL, + fmt.Sprintf("kernel: Out of memory: Killed process %s (%s) total-vm:%dkB, anon-rss:%dkB", pid, proc, 2000000+r.Intn(4000000), 1000000+r.Intn(3000000)), + map[string]string{"unit": "kernel", "pid": pid, "process": proc}) + } +} + +var winEvents = []struct { + id string + provider string + channel string + sev logsv1.Severity + message string + weight int +}{ + {"4624", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "An account was successfully logged on.", 20}, + {"4625", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "An account failed to log on.", 10}, + {"4634", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "An account was logged off.", 14}, + {"4688", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_INFO, "A new process has been created.", 12}, + {"4720", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "A user account was created.", 2}, + {"4740", "Microsoft-Windows-Security-Auditing", "Security", logsv1.Severity_SEVERITY_WARN, "A user account was locked out.", 3}, + {"7036", "Service Control Manager", "System", logsv1.Severity_SEVERITY_INFO, "The Windows Update service entered the running state.", 16}, + {"7031", "Service Control Manager", "System", logsv1.Severity_SEVERITY_ERROR, "The SQL Server (MSSQLSERVER) service terminated unexpectedly.", 3}, + {"1000", "Application Error", "Application", logsv1.Severity_SEVERITY_ERROR, "Faulting application name: ShopSync.exe, version 4.2.1.0, exception code 0xc0000005", 5}, + {"6008", "EventLog", "System", logsv1.Severity_SEVERITY_ERROR, "The previous system shutdown was unexpected.", 1}, + {"41", "Microsoft-Windows-Kernel-Power", "System", logsv1.Severity_SEVERITY_FATAL, "The system has rebooted without cleanly shutting down first.", 1}, +} + +var winWeightTotal int + +func init() { + for _, e := range winEvents { + winWeightTotal += e.weight + } +} + +// eventlogRecord mirrors /hack/windows-fixture's attribute contract +// (winevt.* keys) so a query written against one works against the +// other -- that fixture stays the small correctness check it was built +// as; this one supplies demo volume. +func eventlogRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + n := r.Intn(winWeightTotal) + e := winEvents[0] + for _, cand := range winEvents { + if n -= cand.weight; n < 0 { + e = cand + break + } + } + // A probe window reaches the Windows tier as failed logons too. + if c.bruteForce && r.Float64() < 0.5 { + e = winEvents[1] + } + + attrs := map[string]string{ + "winevt.event_id": e.id, + "winevt.provider": e.provider, + "winevt.channel": e.channel, + "winevt.computer": h.name, + "winevt.record_number": strconv.Itoa(100000 + r.Intn(900000)), + } + msg := e.message + switch e.id { + case "4624", "4625", "4634": + user := pick(r, []string{"SHOP\\svc_sync", "SHOP\\jcoffey", "SHOP\\administrator", "SHOP\\backup"}) + logonType := pick(r, []string{"3", "10", "2"}) + attrs["winevt.target_user"] = user + attrs["winevt.logon_type"] = logonType + src := pick(r, clientIPs) + if e.id == "4625" { + src = pick(r, attackerIPs) + attrs["winevt.status"] = "0xC000006D" + } + attrs["remote_addr"] = src + msg = fmt.Sprintf("%s Account: %s Logon Type: %s Source Network Address: %s", e.message, user, logonType, src) + case "4688": + proc := pick(r, []string{"C:\\Windows\\System32\\cmd.exe", "C:\\Program Files\\ShopSync\\ShopSync.exe", "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"}) + attrs["winevt.process"] = proc + msg = fmt.Sprintf("%s New Process Name: %s", e.message, proc) + case "4740": + user := pick(r, []string{"SHOP\\jcoffey", "SHOP\\svc_sync"}) + attrs["winevt.target_user"] = user + msg = fmt.Sprintf("%s Account Name: %s Caller Computer Name: %s", e.message, user, h.name) + } + return newRecord(h, "eventlog", t, e.sev, msg, attrs) +} + +// primaryRecord dispatches to whichever generator matches this host's +// role. Kept as one switch rather than a func field on host so fleet.go +// stays pure data. +func primaryRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + switch h.service { + case "nginx": + return nginxRecord(h, t, r, c) + case "api": + return apiRecord(h, t, r, c) + case "worker": + return workerRecord(h, t, r, c) + case "postgres": + return postgresRecord(h, t, r, c) + case "redis": + return redisRecord(h, t, r, c) + case "smtp": + return smtpRecord(h, t, r, c) + case "eventlog": + return eventlogRecord(h, t, r, c) + default: + return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "heartbeat", nil) + } +} diff --git a/hack/demo-simulator/fleet.go b/hack/demo-simulator/fleet.go new file mode 100644 index 0000000..ab15293 --- /dev/null +++ b/hack/demo-simulator/fleet.go @@ -0,0 +1,241 @@ +package main + +// The synthetic fleet the demo deployment pretends to be monitoring: a +// small e-commerce shop's infrastructure. Every host here is fictional, +// but the shape is deliberately realistic -- an edge/nginx tier, a +// three-node API tier, background workers, one Postgres, one Redis, one +// Stalwart mail host, two Windows boxes, and one decommissioned host +// left behind on purpose so the Agents page has a genuinely stale row to +// show (see stale below). +// +// One entry here is one *host*, not one agent process: the `agents` +// table is UNIQUE (tenant_id, host) (see +// metadata/migrations/0037_create_agents.sql), so a host maps to exactly +// one agent row, one metrics series, and one heartbeat stream. Log +// records are not bound by that -- a host emits its primary service's +// logs plus, on Linux, the journald `system` stream every real +// deployment also collects. + +type host struct { + name string + service string // primary log service, and the service its agent reports + + // Static context the Hosts page shows alongside utilization (see + // web/src/lib/api.ts's HostMetrics) -- a viewer can't judge "21% CPU" + // without the core count, or "is this normal" without uptime. + os string + kernel string + arch string + cores int + memTotal int64 + diskTot int64 + ipv4 string + ipv6 string + + // Utilization baselines. Each sample wanders around these rather + // than being redrawn independently, so the Hosts page shows a host + // with a personality (a busy API node, an idle cache) instead of the + // same noise everywhere. + cpuBase float64 // mean CPU percent + memFrac float64 // mean fraction of memTotal in use + diskFrac float64 // fraction of diskTot in use at the START of the backfill window + // diskGrowthPerDay pushes diskFrac up over the window -- the "disk + // slowly filling up" story the disk-usage alert rule fires on. Zero + // for every host that isn't part of that story. + diskGrowthPerDay float64 + + // Peak-hour log rates, in events per minute, before the diurnal + // curve and -rate-scale are applied. systemPerMin is the journald + // system stream (sshd/ufw/systemd/kernel), zero on Windows hosts. + eventsPerMin float64 + systemPerMin float64 + + // What this host's agent reports about itself on CheckIn. + agentVersion string + sourceKind string // "journald", "file", "eventlog" + sourceDetail string + batchMax int64 + batchFlushMS int64 + heartbeatMS int64 + + // stale hosts check in exactly once at startup and then go quiet, so + // the Agents page's staleness heuristic (last_seen older than 3x the + // heartbeat interval, floor 5 minutes -- see + // web/src/routes/agents/+page.svelte) flags them a few minutes into + // any demo session. They emit no logs and no metrics either: a host + // whose agent is gone stops producing everything, not just + // heartbeats. + stale bool +} + +const agentVersion = "0.6.2" + +var fleet = []host{ + { + name: "edge-01", service: "nginx", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 100 << 30, + ipv4: "10.0.1.11", ipv6: "2600:3c02::f03c:94ff:fe1a:1101", + cpuBase: 22, memFrac: 0.41, diskFrac: 0.36, + eventsPerMin: 15, systemPerMin: 0.7, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/nginx/access.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "edge-02", service: "nginx", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 100 << 30, + ipv4: "10.0.1.12", ipv6: "2600:3c02::f03c:94ff:fe1a:1102", + cpuBase: 19, memFrac: 0.38, diskFrac: 0.33, + eventsPerMin: 13, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/nginx/access.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "api-01", service: "api", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 160 << 30, + ipv4: "10.0.2.21", ipv6: "2600:3c02::f03c:94ff:fe1a:2101", + cpuBase: 34, memFrac: 0.52, diskFrac: 0.29, + eventsPerMin: 10, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", + batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + }, + { + name: "api-02", service: "api", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 160 << 30, + ipv4: "10.0.2.22", ipv6: "2600:3c02::f03c:94ff:fe1a:2102", + cpuBase: 37, memFrac: 0.57, diskFrac: 0.31, + eventsPerMin: 10, systemPerMin: 0.5, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-api.service", + batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + }, + { + name: "api-03", service: "api", + // One host deliberately a release behind, so the Agents page's + // agent_version column shows a fleet that isn't uniformly + // upgraded -- the normal state of any real fleet. + os: "Debian GNU/Linux 12 (bookworm)", kernel: "6.1.0-25-amd64", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 160 << 30, + ipv4: "10.0.2.23", ipv6: "2600:3c02::f03c:94ff:fe1a:2103", + cpuBase: 41, memFrac: 0.61, diskFrac: 0.44, + eventsPerMin: 9, systemPerMin: 0.5, + agentVersion: "0.5.4", sourceKind: "journald", sourceDetail: "unit=shop-api.service", + batchMax: 1000, batchFlushMS: 3000, heartbeatMS: 60000, + }, + { + name: "worker-01", service: "worker", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, + ipv4: "10.0.3.31", ipv6: "2600:3c02::f03c:94ff:fe1a:3101", + cpuBase: 46, memFrac: 0.63, diskFrac: 0.4, + eventsPerMin: 4.5, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", + batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "worker-02", service: "worker", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 4, memTotal: 8 << 30, diskTot: 200 << 30, + ipv4: "10.0.3.32", ipv6: "2600:3c02::f03c:94ff:fe1a:3102", + cpuBase: 52, memFrac: 0.71, diskFrac: 0.62, + // The one host with a real, visible trend: ~4 points of disk a + // day, so a 7-day backfill window ends with it close to full and + // the "Disk filling up" alert rule has something true to fire on. + diskGrowthPerDay: 0.04, + eventsPerMin: 4.5, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-worker.service", + batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "db-01", service: "postgres", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 500 << 30, + ipv4: "10.0.4.41", ipv6: "2600:3c02::f03c:94ff:fe1a:4101", + cpuBase: 28, memFrac: 0.74, diskFrac: 0.51, + eventsPerMin: 6, systemPerMin: 0.4, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/log/postgresql/postgresql-17-main.log", + batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "cache-01", service: "redis", + os: "Ubuntu 22.04.5 LTS", kernel: "5.15.0-118-generic", arch: "x86_64", + cores: 2, memTotal: 8 << 30, diskTot: 50 << 30, + ipv4: "10.0.4.51", ipv6: "2600:3c02::f03c:94ff:fe1a:5101", + cpuBase: 11, memFrac: 0.58, diskFrac: 0.18, + eventsPerMin: 2, systemPerMin: 0.3, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=redis-server.service", + batchMax: 500, batchFlushMS: 10000, heartbeatMS: 60000, + }, + { + name: "mail-01", service: "smtp", + os: "Debian GNU/Linux 13 (trixie)", kernel: "6.12.9-amd64", arch: "x86_64", + cores: 2, memTotal: 4 << 30, diskTot: 250 << 30, + ipv4: "198.51.100.25", ipv6: "2600:3c06::2000:7dff:fe55:2501", + cpuBase: 14, memFrac: 0.46, diskFrac: 0.57, + eventsPerMin: 6, systemPerMin: 1.2, // internet-facing: more scan/ssh noise than an internal host + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/opt/stalwart/logs/current.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "arm-build-01", service: "worker", + // The fleet's one non-x86 host, so `stats count by arch`-style + // questions and the Hosts page's Architecture row have more than + // one answer in them. + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "aarch64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.5.61", ipv6: "2600:3c02::f03c:94ff:fe1a:6101", + cpuBase: 63, memFrac: 0.55, diskFrac: 0.47, + eventsPerMin: 3, systemPerMin: 0.3, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=buildkite-agent.service", + batchMax: 1000, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "WIN-APP-01", service: "eventlog", + os: "Windows Server 2022 Datacenter", kernel: "10.0.20348", arch: "x86_64", + cores: 4, memTotal: 16 << 30, diskTot: 250 << 30, + ipv4: "10.0.6.71", ipv6: "", + cpuBase: 26, memFrac: 0.64, diskFrac: 0.42, + eventsPerMin: 2.5, systemPerMin: 0, + agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "WIN-SQL-01", service: "eventlog", + os: "Windows Server 2019 Standard", kernel: "10.0.17763", arch: "x86_64", + cores: 8, memTotal: 32 << 30, diskTot: 500 << 30, + ipv4: "10.0.6.72", ipv6: "", + cpuBase: 33, memFrac: 0.78, diskFrac: 0.66, + eventsPerMin: 2, systemPerMin: 0, + agentVersion: "0.5.4", sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "legacy-01", service: "nginx", + os: "Ubuntu 20.04.6 LTS", kernel: "5.4.0-192-generic", arch: "x86_64", + cores: 2, memTotal: 4 << 30, diskTot: 40 << 30, + ipv4: "10.0.1.19", ipv6: "", + cpuBase: 3, memFrac: 0.22, diskFrac: 0.71, + eventsPerMin: 0, systemPerMin: 0, + agentVersion: "0.4.9", sourceKind: "file", sourceDetail: "/var/log/nginx/access.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + stale: true, + }, +} + +// linuxHosts is every host whose agent tails journald or a file -- i.e. +// everything that also produces the `system` service stream. Windows +// hosts produce eventlog records instead, and the stale host produces +// nothing at all. +func linuxHosts() []*host { + var out []*host + for i := range fleet { + h := &fleet[i] + if h.stale || h.service == "eventlog" { + continue + } + out = append(out, h) + } + return out +} diff --git a/hack/demo-simulator/go.mod b/hack/demo-simulator/go.mod new file mode 100644 index 0000000..7636516 --- /dev/null +++ b/hack/demo-simulator/go.mod @@ -0,0 +1,18 @@ +module github.com/cairnobs/cairnobs/hack/demo-simulator + +go 1.25.0 + +replace github.com/cairnobs/cairnobs/proto => ../../proto + +require ( + github.com/cairnobs/cairnobs/proto v0.0.0-00010101000000-000000000000 + google.golang.org/grpc v1.83.0 +) + +require ( + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/hack/demo-simulator/go.sum b/hack/demo-simulator/go.sum new file mode 100644 index 0000000..481b598 --- /dev/null +++ b/hack/demo-simulator/go.sum @@ -0,0 +1,38 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/hack/demo-simulator/incidents.go b/hack/demo-simulator/incidents.go new file mode 100644 index 0000000..b2e1911 --- /dev/null +++ b/hack/demo-simulator/incidents.go @@ -0,0 +1,134 @@ +package main + +import ( + "math" + "time" +) + +// Incidents are what turn a wall of uniform noise into a dataset worth +// clicking around in: a demo viewer who filters to 5xx, or opens the +// Alerts page, should find a *story* (one bad API node, a burst of SSH +// probing, a spam wave) rather than the same flat error rate everywhere. +// Every window below is expressed relative to `origin` -- the moment the +// simulator started, which is also the end of the backfill window -- so +// a freshly reset demo always has its incidents at the same, recent, +// predictable offsets no matter what day it is. +type conditions struct { + // apiErrorRate replaces the API tier's baseline 5xx probability for + // the affected host during an outage window. + apiErrorRate float64 + // latencyMult multiplies API/DB latencies -- an outage that only + // changed status codes without slowing anything down wouldn't look + // like a real one. + latencyMult float64 + // jobFailureRate replaces the worker tier's baseline failure + // probability. An API/database outage doesn't stay in the request + // path: the same failing dependencies take background jobs down with + // them, which is also the only thing that ever makes the job-failure + // alert rule true -- a steady 5% background failure rate is normal + // operation, not an incident. + jobFailureRate float64 + // bruteForce and spamWave switch the system/smtp generators from + // their normal mix to an attack-shaped one for the window. + bruteForce bool + spamWave bool +} + +// Windows, as offsets back from origin. Kept as one table so the story +// is readable in one place and the alert rules in +// /hack/demo-seed/alerts can be written against known-true conditions. +const ( + apiOutageStart = 8 * time.Hour + apiOutageEnd = 6*time.Hour + 30*time.Minute + apiOutageHost = "api-02" + + bruteForceStart = 14 * time.Hour + bruteForceEnd = 13 * time.Hour + + spamWaveStart = 30 * time.Hour + spamWaveEnd = 26 * time.Hour + + // Live mode can't rely on the backfill windows above -- they recede + // into the past as a demo session runs. These recurring bursts keep + // the *present* interesting too, which is what the alert rules + // (evaluating over -5m/-10m windows) actually see: without them, + // every rule would settle into a permanent OK state a few minutes + // after a reset and the Alerts page would never do anything again. + liveErrorBurstPeriod = 47 * time.Minute + liveErrorBurstLen = 5 * time.Minute + liveProbePeriod = 2 * time.Hour + liveProbeLen = 8 * time.Minute + liveSpamPeriod = 3 * time.Hour + liveSpamLen = 10 * time.Minute +) + +func conditionsAt(t, origin time.Time, h *host) conditions { + hostName := h.name + c := conditions{latencyMult: 1} + since := origin.Sub(t) + + // Is an API-tier outage in effect? Two sources feed the same + // handling: the backfilled incident window, and -- in live mode -- + // the recurring burst that keeps the present interesting. + outage := 0.0 + if since <= apiOutageStart && since >= apiOutageEnd { + outage = 0.42 + } + bruteForce := since <= bruteForceStart && since >= bruteForceEnd + spamWave := since <= spamWaveStart && since >= spamWaveEnd + + if t.After(origin) { + // Phases are measured from origin so the first burst of each kind + // lands a predictable few minutes into a demo session rather than + // immediately at reset. + elapsed := t.Sub(origin) + if phase := (elapsed + 10*time.Minute) % liveErrorBurstPeriod; phase < liveErrorBurstLen { + outage = 0.45 + } + if phase := (elapsed + 20*time.Minute) % liveProbePeriod; phase < liveProbeLen { + bruteForce = true + } + if phase := (elapsed + 35*time.Minute) % liveSpamPeriod; phase < liveSpamLen { + spamWave = true + } + } + + if outage > 0 { + switch { + case hostName == apiOutageHost: + c.apiErrorRate = outage + c.latencyMult = 4.5 + case internetFacingName(hostName): + // The edge tier fronts all three API nodes, so roughly a + // third of what it proxies during the outage hits the failing + // one. Without this the outage would be invisible from the + // edge, which isn't how a viewer expects to be able to trace + // it: client-visible 5xx are exactly what makes it an outage + // rather than an internal blip. + c.apiErrorRate = outage / 3 + c.latencyMult = 2 + case hostName == "db-01": + // The database is the *cause*, not a second unrelated + // incident -- a viewer who drills from the API errors into + // the same window on db-01 should find slow queries waiting. + c.latencyMult = 6 + case h.service == "worker": + c.jobFailureRate = 0.35 + c.latencyMult = 2.5 + } + } + c.bruteForce = bruteForce + c.spamWave = spamWave + + return c +} + +// diurnal scales event rates by time of day: a shop's traffic peaks +// mid-afternoon UTC and bottoms out around 04:00, roughly a 3.5x spread. +// Without this every chart is a flat line and the "last 24h" view tells +// a viewer nothing that "last 1h" didn't. +func diurnal(t time.Time) float64 { + hour := float64(t.UTC().Hour()) + float64(t.UTC().Minute())/60 + // Peak at 15:00 UTC, trough at 03:00. + return 0.35 + 0.65*(0.5+0.5*math.Cos((hour-15)/24*2*math.Pi)) +} diff --git a/hack/demo-simulator/main.go b/hack/demo-simulator/main.go new file mode 100644 index 0000000..7bfd5c2 --- /dev/null +++ b/hack/demo-simulator/main.go @@ -0,0 +1,410 @@ +// Command demo-simulator is the demo deployment's whole synthetic world +// in one process: a fictional fleet (see fleet.go) whose agents check in +// over AgentControl, report CPU/memory/disk, and ship realistically +// shaped logs for eight services -- backfilled across a window of +// history first, then continuously in real time for as long as it runs. +// +// Why one long-running process rather than another one-shot fixture: +// three of the demo's features are only convincing if data keeps +// arriving. The Agents page marks a host stale once it stops checking in +// (a one-shot fixture's fleet would go stale minutes after the nightly +// reset); alert rules evaluate over trailing windows like -5m and would +// settle into a permanent OK state against a frozen dataset; and a live +// tail or a "last 15 minutes" dashboard over a dataset that stopped +// growing at 03:00 shows an empty screen. Backfill alone can't fix any +// of those. +// +// It does not replace /hack/benchmark-fixture (volume benchmarking) or +// /hack/windows-fixture (Windows pipeline correctness) -- those stay the +// focused tools they were built as. This one is for the demo. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "math/rand" + "os" + "os/signal" + "sort" + "sync" + "sync/atomic" + "syscall" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + agentv1 "github.com/cairnobs/cairnobs/proto/sentry/agent/v1" + logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1" +) + +// metricsInterval is how often each host samples CPU/memory/disk and +// emits a heartbeat, in both backfill and live mode -- matching the real +// agent's default 60s heartbeat would multiply backfill volume for no +// visible gain on a chart, so history is sampled more coarsely than the +// present. +const ( + backfillMetricsInterval = 5 * time.Minute + liveMetricsInterval = time.Minute + liveTick = 5 * time.Second +) + +func main() { + addr := flag.String("addr", "localhost:4317", "ingest gRPC address") + caFile := flag.String("ca", "../dev-certs/out/ca.pem", "CA cert path") + certFile := flag.String("cert", "../dev-certs/out/client.pem", "client cert path") + keyFile := flag.String("key", "../dev-certs/out/client-key.pem", "client key path") + backfill := flag.Duration("backfill", 168*time.Hour, "how much history to generate before going live; 0 skips backfill") + live := flag.Bool("live", true, "after backfill, keep generating events in real time until terminated") + rateScale := flag.Float64("rate-scale", 0.5, "multiplier on every host's per-minute event rate -- the knob for how much total data a backfill produces") + batchSize := flag.Int("batch-size", 1000, "records per PushBatch call") + concurrency := flag.Int("concurrency", 4, "concurrent PushBatch calls in flight during backfill") + seed := flag.Int64("seed", 0, "random seed; 0 uses the current time") + dryRun := flag.Bool("dry-run", false, "generate the backfill without connecting to ingest and print what it would have sent, then exit") + flag.Parse() + + if *seed == 0 { + *seed = time.Now().UnixNano() + } + + if *dryRun { + runDryRun(context.Background(), time.Now(), *backfill, *rateScale, *seed) + return + } + + tlsConf, err := loadTLSConfig(*caFile, *certFile, *keyFile) + if err != nil { + fmt.Fprintln(os.Stderr, "loading TLS config:", err) + os.Exit(1) + } + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf))) + if err != nil { + fmt.Fprintln(os.Stderr, "dialing ingest:", err) + os.Exit(1) + } + defer conn.Close() + + logs := logsv1.NewLogIngestClient(conn) + control := agentv1.NewAgentControlClient(conn) + + // origin is both the end of the backfill window and the reference + // point every incident window is measured back from, so history and + // live traffic tell one continuous story. + origin := time.Now() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if *backfill > 0 { + runBackfill(ctx, logs, origin, *backfill, *rateScale, *batchSize, *concurrency, *seed) + } + if ctx.Err() != nil { + return + } + if !*live { + return + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); runCheckIns(ctx, control) }() + // metricsAnchor is the oldest moment this run's metrics series + // covers. It has to be the same value in both modes: disk growth and + // uptime are both measured from it, and anchoring live samples at + // `origin` instead would make worker-02's disk usage jump backwards + // (and every uptime reset to near zero) the instant backfill ended. + metricsAnchor := origin.Add(-*backfill) + go func() { defer wg.Done(); runLive(ctx, logs, origin, metricsAnchor, *rateScale, *seed) }() + wg.Wait() + log.Println("demo-simulator stopped") +} + +// runBackfill walks the history window a minute at a time, streaming +// batches to a small pool of pushers as it goes rather than building the +// whole dataset in memory first -- the demo box this runs on has 3GB of +// RAM and a week of history is hundreds of thousands of records. +func runBackfill(ctx context.Context, client logsv1.LogIngestClient, origin time.Time, window time.Duration, rateScale float64, batchSize, concurrency int, seed int64) { + start := origin.Add(-window) + log.Printf("backfilling %s of history (%s .. %s) at rate-scale %.2f", + window, start.UTC().Format(time.RFC3339), origin.UTC().Format(time.RFC3339), rateScale) + + batches := make(chan []*logsv1.LogRecord, concurrency*2) + var sent atomic.Int64 + var failed atomic.Int64 + + var pushers sync.WaitGroup + for i := 0; i < concurrency; i++ { + pushers.Add(1) + go func(worker int) { + defer pushers.Done() + for batch := range batches { + n, err := push(ctx, client, fmt.Sprintf("demo-backfill-%d-%d", worker, sent.Load()), batch) + if err != nil { + if ctx.Err() == nil { + log.Printf("backfill PushBatch failed: %v", err) + } + failed.Add(int64(len(batch))) + continue + } + total := sent.Add(int64(n)) + if total%50000 < int64(batchSize) { + log.Printf("backfill: %d records sent", total) + } + } + }(i) + } + + batch := make([]*logsv1.LogRecord, 0, batchSize) + walkHistory(ctx, origin, window, rateScale, seed, func(rec *logsv1.LogRecord) { + batch = append(batch, rec) + if len(batch) >= batchSize { + batches <- batch + batch = make([]*logsv1.LogRecord, 0, batchSize) + } + }) + if len(batch) > 0 { + batches <- batch + } + close(batches) + pushers.Wait() + + if f := failed.Load(); f > 0 { + log.Printf("backfill complete: %d records sent, %d dropped by failed pushes", sent.Load(), f) + return + } + log.Printf("backfill complete: %d records sent", sent.Load()) +} + +// walkHistory replays the backfill window a minute at a time, handing +// every generated record to emit. Shared by the real backfill and +// -dry-run so the two can never disagree about what a run would produce. +func walkHistory(ctx context.Context, origin time.Time, window time.Duration, rateScale float64, seed int64, emit func(*logsv1.LogRecord)) { + start := origin.Add(-window) + rng := rand.New(rand.NewSource(seed)) + + nextMetrics := start + for minute := start; minute.Before(origin) && ctx.Err() == nil; minute = minute.Add(time.Minute) { + metricsDue := !minute.Before(nextMetrics) + if metricsDue { + nextMetrics = minute.Add(backfillMetricsInterval) + } + for i := range fleet { + h := &fleet[i] + if h.stale { + continue + } + c := conditionsAt(minute, origin, h) + for _, rec := range minuteRecords(h, minute, rng, c, rateScale) { + emit(rec) + } + if metricsDue { + emit(metricsRecord(h, minute, start, rng)) + emit(heartbeatRecord(h, minute)) + } + } + } +} + +// runDryRun generates a backfill without sending it anywhere and reports +// what it would have produced -- the volume/mix tuning knob, so +// -rate-scale and the per-host rates in fleet.go can be adjusted without +// pushing a few hundred thousand records into ClickHouse to find out. +func runDryRun(ctx context.Context, origin time.Time, window time.Duration, rateScale float64, seed int64) { + byService := map[string]int{} + bySeverity := map[string]int{} + total := 0 + walkHistory(ctx, origin, window, rateScale, seed, func(rec *logsv1.LogRecord) { + total++ + byService[rec.GetService()]++ + bySeverity[rec.GetSeverity().String()]++ + }) + + fmt.Printf("dry run: %d records over %s at rate-scale %.2f (%.0f/min average)\n", + total, window, rateScale, float64(total)/window.Minutes()) + fmt.Println("by service:") + for _, k := range sortedKeys(byService) { + fmt.Printf(" %-10s %8d\n", k, byService[k]) + } + fmt.Println("by severity:") + for _, k := range sortedKeys(bySeverity) { + fmt.Printf(" %-22s %8d\n", k, bySeverity[k]) + } +} + +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// minuteRecords generates one host's events for one minute of wall +// clock: its primary service's traffic, plus the journald `system` +// stream every Linux host also ships. +func minuteRecords(h *host, minute time.Time, rng *rand.Rand, c conditions, rateScale float64) []*logsv1.LogRecord { + shape := diurnal(minute) * rateScale + var out []*logsv1.LogRecord + for i := 0; i < countFor(h.eventsPerMin*shape, rng); i++ { + out = append(out, primaryRecord(h, jitter(minute, rng), rng, c)) + } + if h.systemPerMin > 0 { + // System/journald volume rises during a probe window -- that + // burst is the whole point of the Security dashboard's panels. + sysRate := h.systemPerMin + if c.bruteForce && internetFacing(h) { + sysRate *= 12 + } + for i := 0; i < countFor(sysRate*shape, rng); i++ { + out = append(out, systemRecord(h, jitter(minute, rng), rng, c)) + } + } + return out +} + +// countFor turns a fractional per-minute rate into a whole number of +// events, carrying the fraction as a probability so a host rated at 0.4 +// events/min really does produce roughly two events every five minutes +// instead of none at all. +func countFor(rate float64, rng *rand.Rand) int { + n := int(rate) + if rng.Float64() < rate-float64(n) { + n++ + } + return n +} + +func jitter(minute time.Time, rng *rand.Rand) time.Time { + return minute.Add(time.Duration(rng.Int63n(int64(time.Minute)))) +} + +// runLive keeps the present moving: the same generators, driven by a +// ticker instead of a cursor, so trailing-window alert rules, the Agents +// page's staleness heuristic, and any "last 15 minutes" view all have +// something real to read. +func runLive(ctx context.Context, client logsv1.LogIngestClient, origin, metricsAnchor time.Time, rateScale float64, seed int64) { + log.Printf("live mode: generating events every %s", liveTick) + rng := rand.New(rand.NewSource(seed + 1)) + + // Fractional carry per host: at a 5-second tick most hosts are owed + // less than one event per tick, and dropping that remainder every + // time would silently zero out every low-rate stream. + carry := make(map[string]float64, len(fleet)*2) + ticker := time.NewTicker(liveTick) + defer ticker.Stop() + metricsTicker := time.NewTicker(liveMetricsInterval) + defer metricsTicker.Stop() + + tickFraction := liveTick.Minutes() + last := time.Now() + + for { + select { + case <-ctx.Done(): + return + + case now := <-ticker.C: + var batch []*logsv1.LogRecord + for i := range fleet { + h := &fleet[i] + if h.stale { + continue + } + c := conditionsAt(now, origin, h) + shape := diurnal(now) * rateScale * tickFraction + + n := carried(carry, h.name+"/primary", h.eventsPerMin*shape) + for j := 0; j < n; j++ { + batch = append(batch, primaryRecord(h, between(last, now, rng), rng, c)) + } + if h.systemPerMin > 0 { + sysRate := h.systemPerMin + if c.bruteForce && internetFacing(h) { + sysRate *= 12 + } + n := carried(carry, h.name+"/system", sysRate*shape) + for j := 0; j < n; j++ { + batch = append(batch, systemRecord(h, between(last, now, rng), rng, c)) + } + } + } + last = now + if len(batch) == 0 { + continue + } + if _, err := push(ctx, client, fmt.Sprintf("demo-live-%d", now.Unix()), batch); err != nil && ctx.Err() == nil { + log.Printf("live PushBatch failed: %v", err) + } + + case now := <-metricsTicker.C: + var batch []*logsv1.LogRecord + for i := range fleet { + h := &fleet[i] + if h.stale { + continue + } + batch = append(batch, metricsRecord(h, now, metricsAnchor, rng), heartbeatRecord(h, now)) + } + if _, err := push(ctx, client, fmt.Sprintf("demo-metrics-%d", now.Unix()), batch); err != nil && ctx.Err() == nil { + log.Printf("metrics PushBatch failed: %v", err) + } + } + } +} + +// carried accumulates a fractional event count for one stream until it +// crosses 1, then spends the whole part. The leftover is kept, not +// rounded away, so long-run volume matches the configured rate exactly +// rather than drifting low -- at a 5-second tick most streams are owed +// well under one event per tick, and rounding would zero them out. +func carried(carry map[string]float64, key string, rate float64) int { + total := carry[key] + rate + n := int(total) + carry[key] = total - float64(n) + return n +} + +func between(from, to time.Time, rng *rand.Rand) time.Time { + span := to.Sub(from) + if span <= 0 { + return to + } + return from.Add(time.Duration(rng.Int63n(int64(span)))) +} + +func push(ctx context.Context, client logsv1.LogIngestClient, batchID string, records []*logsv1.LogRecord) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + resp, err := client.PushBatch(ctx, &logsv1.PushBatchRequest{BatchId: batchID, Records: records}) + if err != nil { + return 0, err + } + return int(resp.GetAccepted()), nil +} + +func loadTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("reading CA cert %s: %w", caFile, err) + } + caPool := x509.NewCertPool() + if !caPool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("no valid certificates found in %s", caFile) + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("loading client cert/key: %w", err) + } + + return &tls.Config{ + RootCAs: caPool, + Certificates: []tls.Certificate{cert}, + }, nil +} diff --git a/hack/demo-simulator/metrics.go b/hack/demo-simulator/metrics.go new file mode 100644 index 0000000..7209aab --- /dev/null +++ b/hack/demo-simulator/metrics.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "hash/fnv" + "math" + "math/rand" + "strconv" + "time" + + logsv1 "github.com/cairnobs/cairnobs/proto/sentry/logs/v1" +) + +// Metrics records carry the exact attribute contract the Hosts page +// reads (see web/src/lib/api.ts's getHostMetrics/listMetricsHosts): a +// `cairnobs.metrics=true` tag plus utilization and static-context +// fields, shipped as an ordinary tagged LogRecord because the query +// language maps any non-standard field name to attributes['field'] with +// automatic numeric casting -- no separate metrics pipeline exists, by +// design. Heartbeat records use the same trick with +// `cairnobs.heartbeat=true`, which is what the absence-style alert rules +// watch for. + +// phaseOf gives each host its own deterministic offset into the wander +// functions below, so two hosts with the same baseline don't move in +// lockstep across the fleet's charts. +func phaseOf(name string) float64 { + h := fnv.New32a() + _, _ = h.Write([]byte(name)) + return float64(h.Sum32()%1000) / 1000 * 2 * math.Pi +} + +func clamp(v, lo, hi float64) float64 { + return math.Max(lo, math.Min(hi, v)) +} + +// metricsRecord samples host h at time t. backfillStart anchors the +// disk-growth trend so a host that's "filling up" is at its lowest at +// the oldest end of the window and its highest right now, in whichever +// order the samples happen to be generated. +func metricsRecord(h *host, t, backfillStart time.Time, r *rand.Rand) *logsv1.LogRecord { + phase := phaseOf(h.name) + mins := float64(t.Unix()) / 60 + + // Two sine components of different periods plus noise: a slow + // business-hours swell and a faster one, so a CPU chart looks like a + // machine doing work rather than a random walk. + cpu := h.cpuBase * (1 + + 0.45*math.Sin(mins/97+phase) + + 0.2*math.Sin(mins/13+phase*2)) + cpu = clamp(cpu*diurnal(t)+r.NormFloat64()*2.5, 0.4, 99) + + memFrac := clamp(h.memFrac*(1+0.08*math.Sin(mins/211+phase))+r.NormFloat64()*0.01, 0.03, 0.97) + + days := t.Sub(backfillStart).Hours() / 24 + diskFrac := clamp(h.diskFrac+h.diskGrowthPerDay*days+r.NormFloat64()*0.002, 0.02, 0.985) + + // A fixed boot moment per host, far enough back that uptimes read + // like real long-lived servers (and differ from each other). + bootOffset := time.Duration(3+int(phase*17)) * 24 * time.Hour + uptime := int64(t.Add(bootOffset).Sub(backfillStart).Seconds()) + + return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "host metrics", map[string]string{ + "cairnobs.metrics": "true", + "cpu_percent": fmt.Sprintf("%.2f", cpu), + "mem_used_bytes": strconv.FormatInt(int64(float64(h.memTotal)*memFrac), 10), + "mem_total_bytes": strconv.FormatInt(h.memTotal, 10), + "disk_used_bytes": strconv.FormatInt(int64(float64(h.diskTot)*diskFrac), 10), + "disk_total_bytes": strconv.FormatInt(h.diskTot, 10), + "cpu_cores": strconv.Itoa(h.cores), + "os_name": h.os, + "kernel_version": h.kernel, + "arch": h.arch, + "uptime_seconds": strconv.FormatInt(uptime, 10), + "ipv4_addresses": h.ipv4, + "ipv6_addresses": h.ipv6, + }) +} + +func heartbeatRecord(h *host, t time.Time) *logsv1.LogRecord { + return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "agent heartbeat", map[string]string{ + "cairnobs.heartbeat": "true", + "agent_version": h.agentVersion, + }) +}