Merge pull request #10 from Coffey-Labs/fix-demo-seed-sql-panels-and-race

Make the demo reset survive a slow start, and drop the SQL panels

hack/demo-seed/dashboards/directory-dns.json

@@ -49,19 +49,20 @@
       "sort_order": 3
     },
     {
-      "title": "DNS query volume by type",
+      "title": "DNS queries by type and result",
-      "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['dns_type'] AS dns_type, count(*) AS count FROM logs WHERE service = 'bind' GROUP BY bucket, dns_type ORDER BY bucket",
+      "query": "service=bind | stats count by dns_type, dns_rcode | sort -count",
-      "viz_type": "line",
+      "viz_type": "bar",
       "position_x": 0,
       "position_y": 3,
       "width": 12,
       "height": 5,
       "viz_config": {
-        "x_column": "bucket",
+        "x_column": "dns_type",
         "value_column": "count",
-        "series_column": "dns_type"
+        "series_column": "dns_rcode",
+        "stacked": "true"
       },
-      "query_language": "sql",
+      "query_language": "spl",
       "sort_order": 4
     },
     {

hack/demo-seed/dashboards/edge-proxy.json

@@ -49,19 +49,20 @@
       "sort_order": 3
     },
     {
-      "title": "Requests per backend over time",
+      "title": "Requests per backend, by balancer",
-      "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['backend'] AS backend, count(*) AS count FROM logs WHERE service = 'haproxy' GROUP BY bucket, backend ORDER BY bucket",
+      "query": "service=haproxy | stats count by backend, host | sort -count",
-      "viz_type": "line",
+      "viz_type": "bar",
       "position_x": 0,
       "position_y": 3,
       "width": 12,
       "height": 5,
       "viz_config": {
-        "x_column": "bucket",
+        "x_column": "backend",
         "value_column": "count",
-        "series_column": "backend"
+        "series_column": "host",
+        "stacked": "true"
       },
-      "query_language": "sql",
+      "query_language": "spl",
       "sort_order": 4
     },
     {

hack/demo-seed/dashboards/messaging-search.json

@@ -49,19 +49,19 @@
       "sort_order": 3
     },
     {
-      "title": "Queue depth over time",
+      "title": "Queue depth by queue and host",
-      "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['queue'] AS queue, max(toFloat64OrZero(attributes['queue_depth'])) AS depth FROM logs WHERE service = 'rabbitmq' GROUP BY bucket, queue ORDER BY bucket",
+      "query": "service=rabbitmq | stats max(queue_depth) as max_depth by queue, host | sort -max_depth",
-      "viz_type": "line",
+      "viz_type": "bar",
       "position_x": 0,
       "position_y": 3,
       "width": 12,
       "height": 5,
       "viz_config": {
-        "x_column": "bucket",
+        "x_column": "queue",
-        "value_column": "depth",
+        "value_column": "max_depth",
-        "series_column": "queue"
+        "series_column": "host"
       },
-      "query_language": "sql",
+      "query_language": "spl",
       "sort_order": 4
     },
     {

hack/demo-seed/dashboards/platform-kubernetes.json

@@ -49,19 +49,20 @@
       "sort_order": 3
     },
     {
-      "title": "Kubelet events over time",
+      "title": "Kubelet events by kind",
-      "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['event_kind'] AS event_kind, count(*) AS count FROM logs WHERE service = 'kubelet' GROUP BY bucket, event_kind ORDER BY bucket",
+      "query": "service=kubelet | stats count by event_kind, host | sort -count",
-      "viz_type": "line",
+      "viz_type": "bar",
       "position_x": 0,
       "position_y": 3,
       "width": 12,
       "height": 5,
       "viz_config": {
-        "x_column": "bucket",
+        "x_column": "host",
         "value_column": "count",
-        "series_column": "event_kind"
+        "series_column": "event_kind",
+        "stacked": "true"
       },
-      "query_language": "sql",
+      "query_language": "spl",
       "sort_order": 4
     },
     {

hack/demo-seed/dashboards/windows-server-estate.json

@@ -49,19 +49,20 @@
       "sort_order": 3
     },
     {
-      "title": "IIS traffic by site",
+      "title": "IIS requests by site and host",
-      "query": "SELECT toStartOfInterval(timestamp, INTERVAL 30 MINUTE) AS bucket, attributes['site'] AS site, count(*) AS count FROM logs WHERE service = 'iis' GROUP BY bucket, site ORDER BY bucket",
+      "query": "service=iis | stats count by site, host | sort -count",
-      "viz_type": "line",
+      "viz_type": "bar",
       "position_x": 0,
       "position_y": 3,
       "width": 12,
       "height": 5,
       "viz_config": {
-        "x_column": "bucket",
+        "x_column": "site",
         "value_column": "count",
-        "series_column": "site"
+        "series_column": "host",
+        "stacked": "true"
       },
-      "query_language": "sql",
+      "query_language": "spl",
       "sort_order": 4
     },
     {

hack/demo-seed/reset-demo.sh

@@ -77,6 +77,24 @@ SVCTOKEN=$(echo "$EVALUATOR_PASSWORD" | ./bin/cairnobsctl users login alerting-e
 printf 'COMPOSE_PROFILES=single-tenant\nALERTING_SERVICE_TOKEN=%s\n' "$SVCTOKEN" > .env && chmod 600 .env
 docker compose up -d alerting
 
+# Wait for it to answer before posting to it. Writing the .env above
+# changes alerting's environment, so `up -d` *recreates* the container
+# rather than leaving it running -- and the next line used to curl it
+# immediately. On a busy box that lost the race: curl returned nothing,
+# `json.load` got an empty string, and the reset died after `down -v` had
+# already wiped everything, leaving the public demo up, empty, and with
+# the simulator still stopped because the unit only restarts at the very
+# end. Cheap to wait; expensive not to.
+echo "waiting for alerting to answer..."
+for i in $(seq 1 60); do
+  if curl -sf -o /dev/null http://localhost:8081/healthz; then break; fi
+  if [ "$i" -eq 60 ]; then
+    echo "!! alerting did not become ready within 60s -- stopping before the seed" >&2
+    exit 1
+  fi
+  sleep 1
+done
+
 # Three notification targets so the Alerts page shows rules routed to
 # different destinations, the way a real deployment splits ops/security/
 # platform. The URLs are deliberately inert placeholders on a domain