diff --git a/hack/demo-seed/README.md b/hack/demo-seed/README.md index 23584c1..aca4e26 100644 --- a/hack/demo-seed/README.md +++ b/hack/demo-seed/README.md @@ -25,9 +25,9 @@ other code. ## The fleet -Fifty hosts, shaped like an estate rather than a stack: thirty-one Linux, -eighteen Windows, and one Linux host whose agent is gone so the Agents -page has something stale to show. +Fifty-four hosts, shaped like an estate rather than a stack: thirty-five +Linux, eighteen Windows, and one Linux host whose agent is gone so the +Agents page has something stale to show. | Tier | Hosts | |---|---| @@ -35,6 +35,7 @@ page has something stale to show. | Application | `api-01`–`04`, `worker-01`–`03`, `arm-build-01` (aarch64) | | Data | `db-01/02` (Postgres), `mysql-01`, `cache-01/02` (Redis), `mq-01/02` (RabbitMQ), `search-01/02` (Elasticsearch) | | Platform | `k8s-node-01`–`03` (kubelet), `ci-01` (Jenkins), `vault-01`, `ldap-01` (OpenLDAP), `dns-01` (BIND), `backup-01`, `mail-01` | +| Commerce | `shop-mag-01/02` (Magento), `shop-woo-01` (WooCommerce), `pay-01` (payment gateway) | | Windows | `DC-01/02`, `IIS-01`–`03`, `WIN-SQL-01/02`, `EXCH-01/02`, `FS-01/02`, `RDS-01/02`, `WIN-APP-01/02`, `PRINT-01`, `WSUS-01`, `SCCM-01` | The Windows share is the point of the proportions. An enterprise looking @@ -49,9 +50,19 @@ Two hosts carry stories the alert rules fire on and must not be moved: and goes quiet, which is what `agent-legacy-01-unavailable` catches. `api-02` is the host the outage window hits. -**Volume.** Fifty hosts generate about 316 records/minute at +**Two storefronts, on purpose.** Magento and WooCommerce write about the +same events differently, so a commerce panel that groups by service rather +than assuming one shape is the honest way to build one. Both feed `pay-01`, +whose authorisations carry amount, gateway and decline reason -- which is +what lets the Commerce and Payments dashboards answer revenue, average +order value, funnel drop-off and why a card was refused out of the same log +lines the operators are already reading. Declines rise during the seeded +outage window alongside the 5xx rate, because the dependency trouble that +fails requests fails authorisations too. + +**Volume.** Fifty-four hosts generate about 346 records/minute at `-rate-scale 1`, and the nightly reset runs at `RATE_SCALE=0.5` over a -168-hour backfill -- roughly **1.9M records per reset**, against about +168-hour backfill -- roughly **2.2M records per reset**, against about 0.5M when the fleet was twelve hosts. ClickHouse is untroubled by that; what it costs is reset time and disk on the demo box. `RATE_SCALE` is the lever if either becomes a problem, and lowering it keeps every host and diff --git a/hack/demo-seed/alerts/agent-dc-01-unavailable.json.template b/hack/demo-seed/alerts/agent-dc-01-unavailable.json.template new file mode 100644 index 0000000..99a1e9e --- /dev/null +++ b/hack/demo-seed/alerts/agent-dc-01-unavailable.json.template @@ -0,0 +1,11 @@ +{ + "name": "DC-01 not reporting", + "description": "No heartbeat from the primary domain controller. Everything that authenticates depends on this host", + "query": "earliest=-5m host=\"DC-01\" cairnobs.heartbeat=true", + "query_language": "spl", + "condition_type": "absence", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true +} diff --git a/hack/demo-seed/alerts/backup-job-failures.json.template b/hack/demo-seed/alerts/backup-job-failures.json.template new file mode 100644 index 0000000..28562cb --- /dev/null +++ b/hack/demo-seed/alerts/backup-job-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Backup job failures", + "description": "Backup jobs are erroring. Any run of these is worth attention: the failure is silent until the restore", + "query": "service=backup result=error earliest=-1h | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 4 +} diff --git a/hack/demo-seed/alerts/ci-build-failures.json.template b/hack/demo-seed/alerts/ci-build-failures.json.template new file mode 100644 index 0000000..1583ed3 --- /dev/null +++ b/hack/demo-seed/alerts/ci-build-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "CI build failures", + "description": "Builds are failing well above the usual rate, which usually means main is broken rather than any one change being bad", + "query": "service=jenkins result=FAILURE earliest=-30m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/commerce-checkout-errors.json.template b/hack/demo-seed/alerts/commerce-checkout-errors.json.template new file mode 100644 index 0000000..8650cab --- /dev/null +++ b/hack/demo-seed/alerts/commerce-checkout-errors.json.template @@ -0,0 +1,13 @@ +{ + "name": "Checkout errors on WooCommerce", + "description": "Checkout is refusing customers above baseline -- an expired coupon campaign, a shipping rule change, or sessions timing out", + "query": "service=woocommerce event_kind=checkout_error earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 30 +} diff --git a/hack/demo-seed/alerts/commerce-no-orders.json.template b/hack/demo-seed/alerts/commerce-no-orders.json.template new file mode 100644 index 0000000..a373010 --- /dev/null +++ b/hack/demo-seed/alerts/commerce-no-orders.json.template @@ -0,0 +1,11 @@ +{ + "name": "Orders have stopped", + "description": "No order has been placed on either storefront for fifteen minutes. At this estate's volume that is not a quiet spell, it is an outage somebody has not noticed yet", + "query": "event_kind=order earliest=-15m", + "query_language": "spl", + "condition_type": "absence", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true +} diff --git a/hack/demo-seed/alerts/commerce-out-of-stock.json.template b/hack/demo-seed/alerts/commerce-out-of-stock.json.template new file mode 100644 index 0000000..a0fc9b6 --- /dev/null +++ b/hack/demo-seed/alerts/commerce-out-of-stock.json.template @@ -0,0 +1,13 @@ +{ + "name": "Products out of stock", + "description": "Repeated attempts to buy items with nothing on hand: the catalogue and the warehouse disagree, and customers are finding out first", + "query": "service=magento event_kind=out_of_stock earliest=-30m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 40 +} diff --git a/hack/demo-seed/alerts/commerce-storefront-exceptions.json.template b/hack/demo-seed/alerts/commerce-storefront-exceptions.json.template new file mode 100644 index 0000000..2ca620c --- /dev/null +++ b/hack/demo-seed/alerts/commerce-storefront-exceptions.json.template @@ -0,0 +1,13 @@ +{ + "name": "Storefront exceptions", + "description": "Uncaught exceptions in the storefront code. Every one of these is a customer looking at an error page mid-purchase", + "query": "event_kind=exception earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 3, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 14 +} diff --git a/hack/demo-seed/alerts/db-01-no-logs.json.template b/hack/demo-seed/alerts/db-01-no-logs.json.template new file mode 100644 index 0000000..a17bf64 --- /dev/null +++ b/hack/demo-seed/alerts/db-01-no-logs.json.template @@ -0,0 +1,11 @@ +{ + "name": "Primary database silent", + "description": "db-01 has logged nothing for ten minutes. Postgres is never quiet on a live system, so silence means the service or the agent has stopped", + "query": "earliest=-10m host=\"db-01\" service=postgres", + "query_language": "spl", + "condition_type": "absence", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true +} diff --git a/hack/demo-seed/alerts/dns-nxdomain-surge.json.template b/hack/demo-seed/alerts/dns-nxdomain-surge.json.template new file mode 100644 index 0000000..0c7862d --- /dev/null +++ b/hack/demo-seed/alerts/dns-nxdomain-surge.json.template @@ -0,0 +1,13 @@ +{ + "name": "DNS NXDOMAIN surge", + "description": "A jump in non-existent domain lookups: a typo-squat campaign, a misconfigured client, or malware working through a generated domain list", + "query": "service=bind dns_rcode=NXDOMAIN earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 3, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 18 +} diff --git a/hack/demo-seed/alerts/edge-no-traffic.json.template b/hack/demo-seed/alerts/edge-no-traffic.json.template new file mode 100644 index 0000000..78c65ac --- /dev/null +++ b/hack/demo-seed/alerts/edge-no-traffic.json.template @@ -0,0 +1,11 @@ +{ + "name": "Edge tier silent", + "description": "Neither load balancer has logged a request. Either traffic has stopped reaching the estate or both agents are down", + "query": "earliest=-10m service=haproxy", + "query_language": "spl", + "condition_type": "absence", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true +} diff --git a/hack/demo-seed/alerts/elasticsearch-gc-pauses.json.template b/hack/demo-seed/alerts/elasticsearch-gc-pauses.json.template new file mode 100644 index 0000000..3613050 --- /dev/null +++ b/hack/demo-seed/alerts/elasticsearch-gc-pauses.json.template @@ -0,0 +1,13 @@ +{ + "name": "Elasticsearch GC pauses", + "description": "Old-generation collections are running long and often -- the heap is under pressure and search latency will follow", + "query": "service=elasticsearch event_kind=gc earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 5, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 20 +} diff --git a/hack/demo-seed/alerts/exchange-bounces.json.template b/hack/demo-seed/alerts/exchange-bounces.json.template new file mode 100644 index 0000000..58e450c --- /dev/null +++ b/hack/demo-seed/alerts/exchange-bounces.json.template @@ -0,0 +1,13 @@ +{ + "name": "Exchange bounce rate", + "description": "Mail is bouncing above the usual rate: a directory out of step with reality, or a distribution list pointing at deleted mailboxes", + "query": "service=exchange event_kind=bounce earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 20 +} diff --git a/hack/demo-seed/alerts/haproxy-backend-5xx.json.template b/hack/demo-seed/alerts/haproxy-backend-5xx.json.template new file mode 100644 index 0000000..8f739ac --- /dev/null +++ b/hack/demo-seed/alerts/haproxy-backend-5xx.json.template @@ -0,0 +1,13 @@ +{ + "name": "HAProxy backend 5xx surge", + "description": "The load balancers are returning 5xx from their backends well above the ~0.8 per ten minutes a healthy estate produces", + "query": "service=haproxy earliest=-10m | where status>=500 | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 2, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 4 +} diff --git a/hack/demo-seed/alerts/iis-5xx.json.template b/hack/demo-seed/alerts/iis-5xx.json.template new file mode 100644 index 0000000..3cf4251 --- /dev/null +++ b/hack/demo-seed/alerts/iis-5xx.json.template @@ -0,0 +1,13 @@ +{ + "name": "IIS 5xx surge", + "description": "The Windows web tier is failing requests -- normally under one per ten minutes across all three IIS hosts", + "query": "service=iis earliest=-10m | where status>=500 | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 2, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 5 +} diff --git a/hack/demo-seed/alerts/k8s-image-pull-failures.json.template b/hack/demo-seed/alerts/k8s-image-pull-failures.json.template new file mode 100644 index 0000000..b420156 --- /dev/null +++ b/hack/demo-seed/alerts/k8s-image-pull-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Kubernetes image pulls failing", + "description": "Nodes cannot pull images: a registry outage, an expired pull secret, or a tag that was deleted out from under a deployment", + "query": "service=kubelet event_kind=image_pull_failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 5, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 30 +} diff --git a/hack/demo-seed/alerts/k8s-readiness-failures.json.template b/hack/demo-seed/alerts/k8s-readiness-failures.json.template new file mode 100644 index 0000000..0c9b02e --- /dev/null +++ b/hack/demo-seed/alerts/k8s-readiness-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Kubernetes readiness probes failing", + "description": "Readiness probes are failing across the cluster at roughly double the normal rate -- pods are being pulled out of service", + "query": "service=kubelet event_kind=probe_failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 5, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 55 +} diff --git a/hack/demo-seed/alerts/ldap-bind-failures.json.template b/hack/demo-seed/alerts/ldap-bind-failures.json.template new file mode 100644 index 0000000..419ba1a --- /dev/null +++ b/hack/demo-seed/alerts/ldap-bind-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "LDAP bind failures", + "description": "Failed LDAP binds are essentially zero in normal operation, so any run of them is worth waking up for", + "query": "service=openldap event_kind=bind_failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 5 +} diff --git a/hack/demo-seed/alerts/mssql-deadlocks.json.template b/hack/demo-seed/alerts/mssql-deadlocks.json.template new file mode 100644 index 0000000..70fdef5 --- /dev/null +++ b/hack/demo-seed/alerts/mssql-deadlocks.json.template @@ -0,0 +1,13 @@ +{ + "name": "SQL Server deadlocks", + "description": "Transactions are being chosen as deadlock victims well above baseline; something is taking locks in a new order", + "query": "service=mssql event_kind=deadlock earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 3, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/mssql-io-stalls.json.template b/hack/demo-seed/alerts/mssql-io-stalls.json.template new file mode 100644 index 0000000..cf81993 --- /dev/null +++ b/hack/demo-seed/alerts/mssql-io-stalls.json.template @@ -0,0 +1,13 @@ +{ + "name": "SQL Server I/O stalls", + "description": "SQL Server is reporting I/O requests over 15 seconds. The storage under the data files is not keeping up", + "query": "service=mssql event_kind=io_stall earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 5, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 35 +} diff --git a/hack/demo-seed/alerts/mssql-login-failures.json.template b/hack/demo-seed/alerts/mssql-login-failures.json.template new file mode 100644 index 0000000..7f7570c --- /dev/null +++ b/hack/demo-seed/alerts/mssql-login-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "SQL Server login failures", + "description": "Repeated failed logins against SQL Server -- credential stuffing against a service account, or an application with a stale password", + "query": "service=mssql event_kind=login_failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/payments-chargebacks.json.template b/hack/demo-seed/alerts/payments-chargebacks.json.template new file mode 100644 index 0000000..a96ac1c --- /dev/null +++ b/hack/demo-seed/alerts/payments-chargebacks.json.template @@ -0,0 +1,13 @@ +{ + "name": "Card chargebacks", + "description": "Chargebacks above the usual trickle, which is the first visible sign of card testing or a fraud run getting through", + "query": "event_kind=chargeback earliest=-1h | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 8 +} diff --git a/hack/demo-seed/alerts/payments-decline-rate.json.template b/hack/demo-seed/alerts/payments-decline-rate.json.template new file mode 100644 index 0000000..b6ef5f9 --- /dev/null +++ b/hack/demo-seed/alerts/payments-decline-rate.json.template @@ -0,0 +1,13 @@ +{ + "name": "Payment decline rate elevated", + "description": "Declines are running well above the ~7.5% baseline. Either a gateway is unwell or something upstream is sending bad authorisations -- both cost orders directly", + "query": "auth_result=declined earliest=-15m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 5, + "notification_target_id": "__TARGET_OPS__", + "enabled": true, + "comparator": "gt", + "threshold_value": 120 +} diff --git a/hack/demo-seed/alerts/rabbitmq-memory-alarm.json.template b/hack/demo-seed/alerts/rabbitmq-memory-alarm.json.template new file mode 100644 index 0000000..e0f31a6 --- /dev/null +++ b/hack/demo-seed/alerts/rabbitmq-memory-alarm.json.template @@ -0,0 +1,13 @@ +{ + "name": "RabbitMQ memory alarms", + "description": "Brokers are raising memory alarms, which blocks publishers -- the queue is not draining as fast as it is filling", + "query": "service=rabbitmq event_kind=alarm earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 3, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 15 +} diff --git a/hack/demo-seed/alerts/rabbitmq-queue-depth.json.template b/hack/demo-seed/alerts/rabbitmq-queue-depth.json.template new file mode 100644 index 0000000..4187e61 --- /dev/null +++ b/hack/demo-seed/alerts/rabbitmq-queue-depth.json.template @@ -0,0 +1,13 @@ +{ + "name": "RabbitMQ queue backing up", + "description": "A queue has passed 2,500 ready messages. Normal depth is a few hundred; this is the shape of consumers dying rather than traffic rising", + "query": "service=rabbitmq earliest=-15m | stats max(queue_depth) as max_depth", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 300, + "for_minutes": 0, + "notification_target_id": "__TARGET_PLATFORM__", + "enabled": true, + "comparator": "gt", + "threshold_value": 2500 +} diff --git a/hack/demo-seed/alerts/smb-access-denied.json.template b/hack/demo-seed/alerts/smb-access-denied.json.template new file mode 100644 index 0000000..2959a4f --- /dev/null +++ b/hack/demo-seed/alerts/smb-access-denied.json.template @@ -0,0 +1,13 @@ +{ + "name": "SMB access denials", + "description": "Share access is being denied repeatedly -- someone or something is reaching for files it has no rights to", + "query": "service=smb event_kind=share_denied earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/squid-denied-surge.json.template b/hack/demo-seed/alerts/squid-denied-surge.json.template new file mode 100644 index 0000000..0cce96b --- /dev/null +++ b/hack/demo-seed/alerts/squid-denied-surge.json.template @@ -0,0 +1,13 @@ +{ + "name": "Squid blocking outbound requests", + "description": "The outbound proxy is denying requests above baseline -- something on the inside is repeatedly reaching for a blocked destination", + "query": "service=squid status=403 earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 3, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 12 +} diff --git a/hack/demo-seed/alerts/vault-auth-failures.json.template b/hack/demo-seed/alerts/vault-auth-failures.json.template new file mode 100644 index 0000000..9751262 --- /dev/null +++ b/hack/demo-seed/alerts/vault-auth-failures.json.template @@ -0,0 +1,13 @@ +{ + "name": "Vault authentication failures", + "description": "Something is failing to authenticate to Vault. Near-zero normally, and the one place where a handful of failures is already a story", + "query": "service=vault event_kind=auth_failed earliest=-10m | stats count", + "query_language": "spl", + "condition_type": "threshold", + "eval_interval_seconds": 60, + "for_minutes": 0, + "notification_target_id": "__TARGET_SECURITY__", + "enabled": true, + "comparator": "gt", + "threshold_value": 3 +} diff --git a/hack/demo-seed/dashboards/capacity-storage.json b/hack/demo-seed/dashboards/capacity-storage.json new file mode 100644 index 0000000..75fa1f8 --- /dev/null +++ b/hack/demo-seed/dashboards/capacity-storage.json @@ -0,0 +1,106 @@ +{ + "name": "Capacity and storage", + "description": "Disk, memory and the jobs that protect them -- what is filling up and what is backing it up", + "default_earliest": "-7d", + "default_latest": "now", + "panels": [ + { + "title": "Peak disk used", + "query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Backup jobs", + "query": "service=backup event_kind=backup | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Backup failures", + "query": "service=backup result=error | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "SQL log backups", + "query": "service=mssql event_kind=log_backup | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Disk used by host", + "query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used by host | sort -used", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 6, + "viz_config": { + "x_column": "host", + "value_column": "used" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Memory headroom", + "query": "cairnobs.metrics=true | stats max(mem_used_bytes) as peak, max(mem_total_bytes) as total by host | sort -peak | head 15", + "viz_type": "table", + "position_x": 0, + "position_y": 9, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Backup outcomes by job", + "query": "service=backup | stats count by backup_job, result", + "viz_type": "bar", + "position_x": 6, + "position_y": 9, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "backup_job", + "value_column": "count", + "series_column": "result", + "stacked": "true" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Recent backup failures", + "query": "service=backup result=error | sort -timestamp | head 20 | fields timestamp, host, backup_job, message", + "viz_type": "table", + "position_x": 0, + "position_y": 14, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 7 + } + ] +} diff --git a/hack/demo-seed/dashboards/commerce.json b/hack/demo-seed/dashboards/commerce.json new file mode 100644 index 0000000..8e0dd7e --- /dev/null +++ b/hack/demo-seed/dashboards/commerce.json @@ -0,0 +1,164 @@ +{ + "name": "Commerce", + "description": "Orders, revenue and the checkout funnel across both storefronts", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Orders", + "query": "event_kind=order | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Revenue", + "query": "event_kind=order | stats sum(order_total) as revenue", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Average order value", + "query": "event_kind=order | stats avg(order_total) as aov", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Items per order", + "query": "event_kind=order | stats avg(order_items) as items", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Revenue by storefront", + "query": "event_kind=order | stats sum(order_total) as revenue by service | sort -revenue", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "service", + "value_column": "revenue" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Orders by store view", + "query": "service=magento event_kind=order | stats count by store_view | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "store_view", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Checkout funnel", + "query": "service=magento event_kind=checkout | stats count by checkout_step | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "checkout_step", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Payment methods", + "query": "event_kind=order | stats count by payment_method | sort -count", + "viz_type": "top_n", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "payment_method", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Best sellers", + "query": "event_kind=order | stats count by sku | sort -count | head 10", + "viz_type": "top_n", + "position_x": 0, + "position_y": 13, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "sku", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Out of stock", + "query": "service=magento event_kind=out_of_stock | stats count by sku | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 13, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "sku", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 9 + }, + { + "title": "Checkout errors", + "query": "service=woocommerce event_kind=checkout_error | sort -timestamp | head 20 | fields timestamp, host, message", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 10 + }, + { + "title": "Storefront exceptions", + "query": "event_kind=exception | sort -timestamp | head 20 | fields timestamp, host, service, message", + "viz_type": "table", + "position_x": 6, + "position_y": 18, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 11 + } + ] +} diff --git a/hack/demo-seed/dashboards/fleet-health.json b/hack/demo-seed/dashboards/fleet-health.json new file mode 100644 index 0000000..1520a08 --- /dev/null +++ b/hack/demo-seed/dashboards/fleet-health.json @@ -0,0 +1,139 @@ +{ + "name": "Fleet health", + "description": "Fifty hosts at a glance: who is loud, who is quiet, and who is running hot", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Log events", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Errors", + "query": "severity=ERROR | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Warnings", + "query": "severity=WARN | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Fatal", + "query": "severity=FATAL | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Log volume by host", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by host | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 6, + "viz_config": { + "x_column": "host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Busiest CPU", + "query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu by host | sort -avg_cpu | head 12", + "viz_type": "top_n", + "position_x": 0, + "position_y": 9, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "host", + "value_column": "avg_cpu" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Fullest disks", + "query": "cairnobs.metrics=true | stats max(disk_used_bytes) as used by host | sort -used | head 12", + "viz_type": "top_n", + "position_x": 6, + "position_y": 9, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "host", + "value_column": "used" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Severity mix by service", + "query": "cairnobs.metrics!=true | stats count by service, severity", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 14, + "width": 12, + "height": 6, + "viz_config": { + "x_column": "service", + "y_column": "severity", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Agents by version", + "query": "cairnobs.heartbeat=true | stats count by agent_version | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 20, + "width": 6, + "height": 4, + "viz_config": { + "x_column": "agent_version", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Quietest hosts", + "query": "cairnobs.metrics!=true cairnobs.heartbeat!=true | stats count by host | sort count | head 10", + "viz_type": "table", + "position_x": 6, + "position_y": 20, + "width": 6, + "height": 4, + "query_language": "spl", + "sort_order": 9 + } + ] +} diff --git a/hack/demo-seed/dashboards/golden-signals.json b/hack/demo-seed/dashboards/golden-signals.json new file mode 100644 index 0000000..e9541bf --- /dev/null +++ b/hack/demo-seed/dashboards/golden-signals.json @@ -0,0 +1,133 @@ +{ + "name": "Golden signals", + "description": "Traffic, errors, latency and saturation across everything that serves a request", + "default_earliest": "-6h", + "default_latest": "now", + "panels": [ + { + "title": "Requests served", + "query": "| where status>=100 | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "5xx responses", + "query": "| where status>=500 | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "API latency (avg ms)", + "query": "service=api | stats avg(duration_ms) as avg_ms", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Slow Postgres statements", + "query": "service=postgres | where duration_ms>1000 | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Traffic by service and host", + "query": "| where status>=100 | stats count by service, host", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "service", + "value_column": "count", + "series_column": "host", + "stacked": "true" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Where the errors are", + "query": "severity=ERROR | stats count by service, host | sort -count | head 20", + "viz_type": "table", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "API latency by route", + "query": "service=api | stats avg(duration_ms) as avg_ms by route | sort -avg_ms | head 12", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "route", + "value_column": "avg_ms" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Status codes by service", + "query": "| where status>=100 | stats count by service, status", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "service", + "y_column": "status", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Slow operations, wherever they are", + "query": "| where duration_ms>1000 | stats count by service, host | sort -count | head 15", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Saturation by host", + "query": "cairnobs.metrics=true | stats avg(cpu_percent) as avg_cpu, max(mem_used_bytes) as peak_mem by host | sort -avg_cpu | head 15", + "viz_type": "table", + "position_x": 6, + "position_y": 18, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 9 + } + ] +} diff --git a/hack/demo-seed/dashboards/payments.json b/hack/demo-seed/dashboards/payments.json new file mode 100644 index 0000000..e651e21 --- /dev/null +++ b/hack/demo-seed/dashboards/payments.json @@ -0,0 +1,141 @@ +{ + "name": "Payments", + "description": "Authorisations, declines and the gateways behind them -- where money stops moving and why", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Authorisations", + "query": "event_kind=authorization | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Declines", + "query": "auth_result=declined | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Refunds", + "query": "event_kind=refund | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Chargebacks", + "query": "event_kind=chargeback | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Approved and declined by gateway", + "query": "event_kind=authorization | stats count by gateway, auth_result", + "viz_type": "bar", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "gateway", + "value_column": "count", + "series_column": "auth_result", + "stacked": "true" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Why cards are declined", + "query": "auth_result=declined | stats count by decline_reason | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "decline_reason", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Declined value", + "query": "auth_result=declined | stats sum(amount) as lost by gateway | sort -lost", + "viz_type": "top_n", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "gateway", + "value_column": "lost" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "Gateway latency", + "query": "event_kind=authorization | stats avg(duration_ms) as avg_ms by gateway | sort -avg_ms", + "viz_type": "bar", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "gateway", + "value_column": "avg_ms" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Decline reasons by gateway", + "query": "auth_result=declined | stats count by gateway, decline_reason", + "viz_type": "heatmap", + "position_x": 0, + "position_y": 13, + "width": 12, + "height": 5, + "viz_config": { + "x_column": "gateway", + "y_column": "decline_reason", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Chargebacks and refunds", + "query": "service=payments | where event_kind=chargeback | sort -timestamp | head 20 | fields timestamp, gateway, amount, message", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 9 + } + ] +} diff --git a/hack/demo-seed/dashboards/security-posture.json b/hack/demo-seed/dashboards/security-posture.json new file mode 100644 index 0000000..77eb95d --- /dev/null +++ b/hack/demo-seed/dashboards/security-posture.json @@ -0,0 +1,145 @@ +{ + "name": "Security posture", + "description": "Authentication, authorisation and the outbound edge -- every place a credential or a request gets refused", + "default_earliest": "-24h", + "default_latest": "now", + "panels": [ + { + "title": "Failed SSH", + "query": "service=system auth_result=failed | stats count", + "viz_type": "single_stat", + "position_x": 0, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 0 + }, + { + "title": "Failed Windows logons", + "query": "service=eventlog winevt.event_id=4625 | stats count", + "viz_type": "single_stat", + "position_x": 3, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 1 + }, + { + "title": "Failed SQL logins", + "query": "service=mssql event_kind=login_failed | stats count", + "viz_type": "single_stat", + "position_x": 6, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 2 + }, + { + "title": "Denied share access", + "query": "service=smb event_kind=share_denied | stats count", + "viz_type": "single_stat", + "position_x": 9, + "position_y": 0, + "width": 3, + "height": 3, + "query_language": "spl", + "sort_order": 3 + }, + { + "title": "Source addresses probing SSH", + "query": "service=system auth_result=failed | stats count by remote_addr | sort -count | head 12", + "viz_type": "top_n", + "position_x": 0, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "remote_addr", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 4 + }, + { + "title": "Accounts targeted on Windows", + "query": "service=eventlog winevt.event_id=4625 | stats count by winevt.target_user | sort -count", + "viz_type": "bar", + "position_x": 6, + "position_y": 3, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "winevt.target_user", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 5 + }, + { + "title": "Blocked outbound destinations", + "query": "service=squid status=403 | stats count by dest_host | sort -count", + "viz_type": "bar", + "position_x": 0, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "x_column": "dest_host", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 6 + }, + { + "title": "NXDOMAIN by client", + "query": "service=bind dns_rcode=NXDOMAIN | stats count by remote_addr | sort -count | head 12", + "viz_type": "top_n", + "position_x": 6, + "position_y": 8, + "width": 6, + "height": 5, + "viz_config": { + "label_column": "remote_addr", + "value_column": "count" + }, + "query_language": "spl", + "sort_order": 7 + }, + { + "title": "Failed LDAP binds", + "query": "service=openldap event_kind=bind_failed | sort -timestamp | head 20 | fields timestamp, host, bind_dn, remote_addr, message", + "viz_type": "table", + "position_x": 0, + "position_y": 13, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 8 + }, + { + "title": "Vault authentication failures", + "query": "service=vault event_kind=auth_failed | sort -timestamp | head 20 | fields timestamp, host, vault_path, remote_addr, message", + "viz_type": "table", + "position_x": 6, + "position_y": 13, + "width": 6, + "height": 5, + "query_language": "spl", + "sort_order": 9 + }, + { + "title": "Account lockouts", + "query": "service=eventlog winevt.event_id=4740 | sort -timestamp | head 20 | fields timestamp, host, winevt.target_user, message", + "viz_type": "table", + "position_x": 0, + "position_y": 18, + "width": 12, + "height": 5, + "query_language": "spl", + "sort_order": 10 + } + ] +} diff --git a/hack/demo-simulator/events.go b/hack/demo-simulator/events.go index 7b98a7c..5e6bfdd 100644 --- a/hack/demo-simulator/events.go +++ b/hack/demo-simulator/events.go @@ -679,6 +679,12 @@ func primaryRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.Log return exchangeRecord(h, t, r, c) case "smb": return smbRecord(h, t, r, c) + case "magento": + return magentoRecord(h, t, r, c) + case "woocommerce": + return wooRecord(h, t, r, c) + case "payments": + return paymentsRecord(h, t, r, c) default: return newRecord(h, h.service, t, logsv1.Severity_SEVERITY_INFO, "heartbeat", nil) } @@ -1057,3 +1063,236 @@ func smbRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogReco map[string]string{"share": share, "winevt.target_user": user, "event_kind": "share_denied"}) } } + +// --------------------------------------------------------------------- +// Commerce. +// +// Two storefronts and the gateway behind both, because the questions a +// business asks of its logs are not the questions an operator asks, and +// a demo that only answers the second one is only half a demo. Orders, +// revenue, average order value, where checkout is losing people and why +// a card was declined are all in the log line rather than in a separate +// metrics system -- which is the argument for keeping the two together. +// +// Two platforms rather than one on purpose: Magento and WooCommerce +// write differently about the same events, so a panel that groups by +// service rather than assuming one shape is the honest way to build one. +// --------------------------------------------------------------------- + +var ( + storeViews = []string{"uk", "us", "de", "fr"} + + skus = []struct { + sku, name string + price float64 + }{ + {"CH-1042", "Aeron-style task chair", 489.00}, + {"DK-2201", "Standing desk 160x80", 629.00}, + {"MN-3310", "27\" 4K monitor", 379.99}, + {"KB-4407", "Mechanical keyboard, tactile", 129.50}, + {"MS-5120", "Vertical ergonomic mouse", 74.95}, + {"LT-6003", "Desk lamp, warm CCT", 59.00}, + {"CB-7788", "Cable management tray", 24.99}, + {"HS-8890", "Noise-cancelling headset", 219.00}, + } + + checkoutSteps = []string{"cart", "shipping", "payment", "review", "placed"} + + gateways = []string{"stripe", "adyen", "paypal"} + + declineReasons = []struct { + code, text string + weight int + }{ + {"insufficient_funds", "Insufficient funds", 30}, + {"do_not_honor", "Do not honour", 22}, + {"expired_card", "Expired card", 14}, + {"incorrect_cvc", "Incorrect CVC", 12}, + {"lost_or_stolen", "Lost or stolen card", 6}, + {"3ds_failed", "3-D Secure authentication failed", 16}, + } + declineWeightTotal int + + paymentMethods = []string{"card", "paypal", "apple_pay", "klarna"} +) + +func init() { + for _, d := range declineReasons { + declineWeightTotal += d.weight + } +} + +func pickDecline(r *rand.Rand) (string, string) { + n := r.Intn(declineWeightTotal) + for _, d := range declineReasons { + if n -= d.weight; n < 0 { + return d.code, d.text + } + } + return declineReasons[0].code, declineReasons[0].text +} + +// orderTotal builds a basket rather than drawing a number, so average +// order value moves the way a real one does -- driven by what is in the +// cart, not by a distribution somebody chose. +func orderTotal(r *rand.Rand) (float64, int, string) { + items := 1 + r.Intn(4) + total := 0.0 + first := "" + for i := 0; i < items; i++ { + s := skus[r.Intn(len(skus))] + qty := 1 + if r.Float64() < 0.18 { + qty = 2 + } + total += s.price * float64(qty) + if i == 0 { + first = s.sku + } + } + return total, items, first +} + +func magentoRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + store := pick(r, storeViews) + switch n := r.Intn(100); { + case n < 34: + // Checkout progress. The funnel narrows towards `placed`, which is + // what makes a "where are we losing people" panel say anything. + step := checkoutSteps[0] + switch f := r.Float64(); { + case f < 0.34: + step = "cart" + case f < 0.58: + step = "shipping" + case f < 0.76: + step = "payment" + case f < 0.88: + step = "review" + default: + step = "placed" + } + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("checkout step reached: %s quote_id=%d store=%s", step, 400000+r.Intn(99999), store), + map[string]string{"checkout_step": step, "store_view": store, "event_kind": "checkout"}) + case n < 58: + total, items, sku := orderTotal(r) + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Order placed: increment_id=%d grand_total=%.2f items=%d store=%s method=%s", + 2000000000+r.Intn(99999999), total, items, store, pick(r, paymentMethods)), + map[string]string{ + "event_kind": "order", "order_total": fmt.Sprintf("%.2f", total), + "order_items": strconv.Itoa(items), "sku": sku, "store_view": store, + "currency": "GBP", "payment_method": pick(r, paymentMethods), + }) + case n < 72: + s := skus[r.Intn(len(skus))] + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Product viewed: sku=%s name=%q store=%s", s.sku, s.name, store), + map[string]string{"event_kind": "product_view", "sku": s.sku, "store_view": store}) + case n < 82: + idx := pick(r, []string{"catalog_product_price", "cataloginventory_stock", "catalogsearch_fulltext", "customer_grid"}) + dur := 4 + r.Intn(180) + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Index %s has been rebuilt successfully in %02d:%02d:%02d", idx, 0, dur/60, dur%60), + map[string]string{"event_kind": "reindex", "indexer": idx, "duration_ms": strconv.Itoa(dur * 1000)}) + case n < 90: + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Cron group %s finished, %d jobs run", pick(r, []string{"default", "index", "consumers"}), 1+r.Intn(20)), + map[string]string{"event_kind": "cron", "store_view": store}) + case n < 96: + s := skus[r.Intn(len(skus))] + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("Not enough items for sale: sku=%s requested=%d on_hand=%d", s.sku, 1+r.Intn(3), r.Intn(2)), + map[string]string{"event_kind": "out_of_stock", "sku": s.sku, "store_view": store}) + default: + return newRecord(h, "magento", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("main.CRITICAL: Uncaught TypeError in %s: Argument #1 must be of type Quote, null given", + pick(r, []string{"Magento/Quote/Model/QuoteManagement.php", "Magento/Checkout/Model/Session.php", "Magento/Sales/Model/Order.php"})), + map[string]string{"event_kind": "exception", "store_view": store}) + } +} + +func wooRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + switch n := r.Intn(100); { + case n < 38: + total, items, sku := orderTotal(r) + status := pick(r, []string{"processing", "completed", "on-hold"}) + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Order #%d status changed to %s (total %.2f, %d items)", 30000+r.Intn(9999), status, total, items), + map[string]string{ + "event_kind": "order", "order_status": status, "order_total": fmt.Sprintf("%.2f", total), + "order_items": strconv.Itoa(items), "sku": sku, "currency": "GBP", + "payment_method": pick(r, paymentMethods), + }) + case n < 58: + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("REST API request: GET /wp-json/wc/v3/products?per_page=%d served in %dms", 10+r.Intn(90), 20+r.Intn(600)), + map[string]string{"event_kind": "api", "duration_ms": strconv.Itoa(20 + r.Intn(600))}) + case n < 74: + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Scheduled action completed: %s", pick(r, []string{"woocommerce_cleanup_sessions", "wc_admin_unsnooze_admin_notes", "woocommerce_scheduled_sales"})), + map[string]string{"event_kind": "cron"}) + case n < 84: + s := skus[r.Intn(len(skus))] + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("Stock reduced for %s: %d -> %d", s.sku, 5+r.Intn(40), r.Intn(5)), + map[string]string{"event_kind": "stock", "sku": s.sku}) + case n < 93: + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("Checkout error: %s", pick(r, []string{ + "Invalid billing postcode", "Coupon \"WELCOME10\" has expired", + "Shipping method not available for this address", "Session expired before payment", + })), + map[string]string{"event_kind": "checkout_error"}) + default: + return newRecord(h, "woocommerce", t, logsv1.Severity_SEVERITY_ERROR, + "PHP Fatal error: Allowed memory size of 268435456 bytes exhausted in class-wc-order.php", + map[string]string{"event_kind": "exception"}) + } +} + +func paymentsRecord(h *host, t time.Time, r *rand.Rand, c conditions) *logsv1.LogRecord { + gw := pick(r, gateways) + total, _, _ := orderTotal(r) + took := int(float64(90+r.Intn(900)) * c.latencyMult) + + // Declines rise with the outage window: the same dependency trouble + // that fails API requests fails authorisations, which is what makes + // the decline-rate rule true at the same time as the 5xx one. + declineRate := 0.075 + if c.apiErrorRate > 0 { + declineRate = 0.28 + } + switch { + case r.Float64() < declineRate: + code, text := pickDecline(r) + return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_WARN, + fmt.Sprintf("authorization declined gateway=%s amount=%.2f currency=GBP reason=%s (%s) latency=%dms", gw, total, code, text, took), + map[string]string{ + "event_kind": "authorization", "auth_result": "declined", "gateway": gw, + "decline_reason": code, "amount": fmt.Sprintf("%.2f", total), + "currency": "GBP", "duration_ms": strconv.Itoa(took), + }) + case r.Float64() < 0.05: + return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("refund issued gateway=%s amount=%.2f currency=GBP reason=%s", gw, total/2, pick(r, []string{"customer_request", "item_returned", "duplicate_charge"})), + map[string]string{"event_kind": "refund", "gateway": gw, "amount": fmt.Sprintf("%.2f", total/2), "currency": "GBP"}) + case r.Float64() < 0.02: + return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_ERROR, + fmt.Sprintf("chargeback received gateway=%s amount=%.2f currency=GBP network_reason=fraud", gw, total), + map[string]string{"event_kind": "chargeback", "gateway": gw, "amount": fmt.Sprintf("%.2f", total), "currency": "GBP"}) + case r.Float64() < 0.10: + return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("3-D Secure challenge issued gateway=%s amount=%.2f currency=GBP", gw, total), + map[string]string{"event_kind": "3ds_challenge", "gateway": gw, "amount": fmt.Sprintf("%.2f", total), "currency": "GBP"}) + default: + return newRecord(h, "payments", t, logsv1.Severity_SEVERITY_INFO, + fmt.Sprintf("authorization approved gateway=%s amount=%.2f currency=GBP latency=%dms", gw, total, took), + map[string]string{ + "event_kind": "authorization", "auth_result": "approved", "gateway": gw, + "amount": fmt.Sprintf("%.2f", total), "currency": "GBP", + "duration_ms": strconv.Itoa(took), + }) + } +} diff --git a/hack/demo-simulator/fleet.go b/hack/demo-simulator/fleet.go index db4b0ee..6321b32 100644 --- a/hack/demo-simulator/fleet.go +++ b/hack/demo-simulator/fleet.go @@ -580,6 +580,46 @@ var fleet = []host{ agentVersion: agentVersion, sourceKind: "eventlog", sourceDetail: "channels=Security,System,Application", batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, }, + { + name: "shop-mag-01", service: "magento", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 32 << 30, diskTot: 400 << 30, + ipv4: "10.0.9.11", ipv6: "2600:3c02::f03c:94ff:fe1a:9011", + cpuBase: 51, memFrac: 0.69, diskFrac: 0.46, + eventsPerMin: 26, systemPerMin: 0.9, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/shop/var/log/system.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "shop-mag-02", service: "magento", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 16, memTotal: 32 << 30, diskTot: 400 << 30, + ipv4: "10.0.9.12", ipv6: "2600:3c02::f03c:94ff:fe1a:9012", + cpuBase: 48, memFrac: 0.67, diskFrac: 0.44, + eventsPerMin: 24, systemPerMin: 0.9, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/shop/var/log/system.log", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "shop-woo-01", service: "woocommerce", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 200 << 30, + ipv4: "10.0.9.21", ipv6: "2600:3c02::f03c:94ff:fe1a:9021", + cpuBase: 37, memFrac: 0.58, diskFrac: 0.39, + eventsPerMin: 17, systemPerMin: 0.7, + agentVersion: agentVersion, sourceKind: "file", sourceDetail: "/var/www/woo/wp-content/uploads/wc-logs/", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, + { + name: "pay-01", service: "payments", + os: "Ubuntu 24.04.1 LTS", kernel: "6.8.0-45-generic", arch: "x86_64", + cores: 8, memTotal: 16 << 30, diskTot: 120 << 30, + ipv4: "10.0.9.31", ipv6: "2600:3c02::f03c:94ff:fe1a:9031", + cpuBase: 23, memFrac: 0.47, diskFrac: 0.28, + eventsPerMin: 19, systemPerMin: 0.6, + agentVersion: agentVersion, sourceKind: "journald", sourceDetail: "unit=shop-payments.service", + batchMax: 500, batchFlushMS: 5000, heartbeatMS: 60000, + }, { name: "legacy-01", service: "nginx", os: "Ubuntu 20.04.6 LTS", kernel: "5.4.0-192-generic", arch: "x86_64",