Monitoring: alerts over metrics, by event and by email (MON-25 to MON-30)

Every minute the node that calculates metrics evaluates each enabled
x:Alert, read from the registry, with metric('name') or the underscore
name. An alert fires when its condition turns true: it emits
telemetry.alert-event with the rendered message, and queues one email per
recipient through the outbound queue, placeholders filled and
Auto-Submitted set. An alert naming an unknown metric is refused on save.
The shared alerts suite runs; every telemetry suite is un-gated.

crates/services/src/task_manager/scheduler.rs

@@ -42,6 +42,8 @@ enum Event {
     RenewNodeIdLease,
     // inbuxa: MON-4: metric history
     StoreMetrics,
+    // inbuxa: MON-25: alert evaluation
+    EvaluateAlerts,
 }
 
 /// When the next metric-history tick is due (MON-4), read from the registry
@@ -126,6 +128,9 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
             // Calculate expensive metrics
             queue.schedule(Instant::now(), Event::CalculateMetrics);
 
+            // inbuxa: MON-25: alerts every minute
+            queue.schedule(Instant::now() + Duration::from_secs(60), Event::EvaluateAlerts);
+
             // inbuxa: MON-4: metric history on its own schedule
             queue.schedule(
                 Instant::now() + metrics_collection_delay(&server).await,
@@ -325,6 +330,43 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
                             );
                         });
                     }
+                    // inbuxa: MON-25 to MON-30: on the node that calculates metrics;
+                    // a failure is logged and tried again next minute (MON-37)
+                    Event::EvaluateAlerts => {
+                        queue.schedule(
+                            Instant::now() + Duration::from_secs(60),
+                            Event::EvaluateAlerts,
+                        );
+                        if server.core.network.roles.metrics_calculate {
+                            let server = server.clone();
+                            tokio::spawn(async move {
+                                match server.process_alerts().await {
+                                    Ok(messages) => {
+                                        use smtp::reporting::send::MtaReportSend;
+                                        for message in messages {
+                                            server
+                                                .send_autogenerated(
+                                                    message.from.clone(),
+                                                    message.to.iter(),
+                                                    message.body,
+                                                    None,
+                                                    0,
+                                                )
+                                                .await;
+                                            trc::event!(
+                                                Telemetry(TelemetryEvent::AlertMessage),
+                                                From = message.from,
+                                                To = message.to,
+                                            );
+                                        }
+                                    }
+                                    Err(err) => {
+                                        trc::error!(err.details("Failed to evaluate alerts"));
+                                    }
+                                }
+                            });
+                        }
+                    }
                     // inbuxa: MON-4: every node writes its own samples
                     Event::StoreMetrics => {
                         queue.schedule(
@@ -431,6 +473,7 @@ impl Event {
             Event::TrainSpamClassifier => "trainSpamClassifier",
             Event::RenewNodeIdLease => "renewNodeIdLease",
             Event::StoreMetrics => "storeMetrics",
+            Event::EvaluateAlerts => "evaluateAlerts",
         }
     }
 }