Monitoring: metric history, one edition of metrics, and x:Metric over the stored samples (MON-4 to MON-7, MON-9, MON-17, MON-39)

Every node writes a sample per metric on metricsCollectionInterval:
counters as the increase since its last sample, gauges always, histograms
as totals when changed. Samples are x:Metric in the registry's encoding
under the telemetry key class, ids time-ordered. x:Metric/get and /query
read them with metric and timestamp filters and full paging, hide what's
past holdMetricsFor, and the data purge deletes it. The is_enterprise
split is gone, so every gauge and histogram is collected and exported, and
queue.count is set from the queue itself. The shared metrics suite runs.

crates/services/src/task_manager/maintenance.rs

@@ -235,6 +235,20 @@ async fn store_maintenance(
             .await
             .caused_by(trc::location!())?;
 
+            use common::telemetry::metrics::store::MetricsStore;
+            // inbuxa: MON-17, MON-38: history past its retention goes; a
+            // failure leaves it for the next run
+            let retention = common::telemetry::metrics::store::retention(server).await;
+            if let Some(keep) = retention.hold_metrics_for
+                && !server.metrics_store().is_none()
+                && let Err(err) = server
+                    .metrics_store()
+                    .purge_metrics(keep.into_inner())
+                    .await
+            {
+                trc::error!(err.details("Failed to purge metric history"));
+            }
+
             trc::event!(
                 Store(StoreEvent::DataStorePurged),
                 Elapsed = started.elapsed()

crates/services/src/task_manager/scheduler.rs

@@ -40,6 +40,19 @@ enum Event {
     CalculateMetrics,
     TrainSpamClassifier,
     RenewNodeIdLease,
+    // inbuxa: MON-4: metric history
+    StoreMetrics,
+}
+
+/// When the next metric-history tick is due (MON-4), read from the registry
+/// so a change needs no reload (MON-3).
+async fn metrics_collection_delay(server: &common::Server) -> Duration {
+    utils::cron::SimpleCron::from(
+        common::telemetry::metrics::store::retention(server)
+            .await
+            .metrics_collection_interval,
+    )
+    .time_to_next()
 }
 
 #[derive(Default)]
@@ -113,6 +126,12 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
             // Calculate expensive metrics
             queue.schedule(Instant::now(), Event::CalculateMetrics);
 
+            // inbuxa: MON-4: metric history on its own schedule
+            queue.schedule(
+                Instant::now() + metrics_collection_delay(&server).await,
+                Event::StoreMetrics,
+            );
+
         }
 
 
@@ -210,13 +229,9 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
                             if roles.metrics_push {
                                 let otel = otel.clone();
 
-
-                                #[cfg(not(feature = "enterprise"))]
-                                let is_enterprise = false;
-
                                 tokio::spawn(async move {
                                     let elapsed = Instant::now();
-                                    otel.push_metrics(is_enterprise, start_time).await;
+                                    otel.push_metrics(start_time).await;
 
                                     trc::event!(
                                         Telemetry(TelemetryEvent::MetricsPushed),
@@ -244,6 +259,16 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
                         tokio::spawn(async move {
                             let elapsed = Instant::now();
                             if server.core.network.roles.metrics_calculate {
+                                // inbuxa: MON-7: the queue gauge from the queue itself,
+                                // so it's right after a restart
+                                match server.total_queued_messages().await {
+                                    Ok(total) => {
+                                        Collector::update_gauge(MetricType::QueueCount, total);
+                                    }
+                                    Err(err) => {
+                                        trc::error!(err.details("Failed to count queued messages"));
+                                    }
+                                }
 
                                 if update_other_metrics {
                                     match server.total_accounts().await {
@@ -300,6 +325,17 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
                             );
                         });
                     }
+                    // inbuxa: MON-4: every node writes its own samples
+                    Event::StoreMetrics => {
+                        queue.schedule(
+                            Instant::now() + metrics_collection_delay(&server).await,
+                            Event::StoreMetrics,
+                        );
+                        let server = server.clone();
+                        tokio::spawn(async move {
+                            server.store_metrics().await;
+                        });
+                    }
                     Event::TrainSpamClassifier => {
                         if let Some(train_frequency) = server
                             .core
@@ -394,6 +430,7 @@ impl Event {
             Event::CalculateMetrics => "calculateMetrics",
             Event::TrainSpamClassifier => "trainSpamClassifier",
             Event::RenewNodeIdLease => "renewNodeIdLease",
+            Event::StoreMetrics => "storeMetrics",
         }
     }
 }