Add agent inventory, management, and remote config
Extends the heartbeat mechanism with a second gRPC service on the same mTLS channel (AgentControl.CheckIn, agent-initiated on the existing heartbeat ticker -- still push-only, no inbound port on any agent) so an agent reports its running config and can pick up an operator-set override. A new web UI section (/agents) lists every agent that's checked in, shows its reported config, and lets an operator edit a narrow, deliberately-scoped subset remotely: batch/heartbeat tuning, and (journald sources only) the unit filter. TLS material and the ingest endpoint are never reportable or remotely editable, by proto shape rather than a validation rule -- a bad or malicious edit there could permanently strand an agent or redirect where its logs go, unlike every other editable field, which only degrades behavior. An override lives only in the agent's memory (agent.toml is never rewritten) and re-syncs on the agent's own schedule; changing the journald filter aborts and respawns the source task since there's no other way to change what's being tailed. Building the hot-reload path surfaced a real, independent, pre-existing bug: shutdown was using poll_timeout(), which only drains once flush_interval has elapsed, silently dropping anything buffered more recently on every graceful shutdown that landed between flushes -- fixed with a new unconditional Batcher::flush_all(), now used at both shutdown and hot-reload. Verified live end-to-end against a real stack: an edited heartbeat interval changed a running agent's actual send cadence within one check-in cycle (confirmed by the real timestamps landing in ClickHouse), and an edited journald filter triggered a real source restart, both reflected back in the next reported-config snapshot. See /docs/agent-management-design.md.
This commit is contained in:
@@ -473,3 +473,55 @@ export function createNotificationTarget(input: {
|
||||
return alertingRequest('/targets', { method: 'POST', body: JSON.stringify(input) });
|
||||
}
|
||||
|
||||
// ---- Agent inventory + remote config ----
|
||||
// See /docs/agent-management-design.md. An agent only appears here
|
||||
// after it's checked in at least once (GET /agents/{host} 404s until
|
||||
// then) -- there's no "pre-register a host" step, inventory is purely
|
||||
// observed from real check-ins.
|
||||
export type ConfigOverride = {
|
||||
batch_max_size?: number;
|
||||
batch_flush_interval_ms?: number;
|
||||
heartbeat_enabled?: boolean;
|
||||
heartbeat_interval_ms?: number;
|
||||
journald_unit?: string;
|
||||
};
|
||||
|
||||
export type Agent = {
|
||||
id: string;
|
||||
tenant_id: string;
|
||||
host: string;
|
||||
service: string;
|
||||
agent_version: string;
|
||||
source_kind: string;
|
||||
source_detail: string;
|
||||
batch_max_size: number;
|
||||
batch_flush_interval_ms: number;
|
||||
heartbeat_enabled: boolean;
|
||||
heartbeat_interval_ms: number;
|
||||
first_seen_at: string;
|
||||
last_seen_at: string;
|
||||
desired_override?: ConfigOverride;
|
||||
desired_override_version?: string;
|
||||
applied_override_version: string;
|
||||
pending: boolean;
|
||||
updated_by?: string;
|
||||
};
|
||||
|
||||
export function listAgents(): Promise<Agent[]> {
|
||||
return request<Agent[]>('/agents').then((a) => a ?? []);
|
||||
}
|
||||
|
||||
export function getAgent(host: string): Promise<Agent> {
|
||||
return request(`/agents/${encodeURIComponent(host)}`);
|
||||
}
|
||||
|
||||
export function setAgentConfig(host: string, override: ConfigOverride): Promise<Agent> {
|
||||
return request(`/agents/${encodeURIComponent(host)}/config`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(override)
|
||||
});
|
||||
}
|
||||
|
||||
export function clearAgentConfig(host: string): Promise<void> {
|
||||
return request(`/agents/${encodeURIComponent(host)}/config`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
{ href: '/dashboards', label: 'Dashboards', icon: '▤' },
|
||||
{ href: '/alerts', label: 'Alerts', icon: '▲' },
|
||||
{ href: '/data-sources', label: 'Data Sources', icon: '◈' },
|
||||
{ href: '/agents', label: 'Agents', icon: '●' },
|
||||
{ href: '/settings', label: 'Settings', icon: '⚙' }
|
||||
];
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
<script lang="ts">
|
||||
import { listAgents, type Agent } from '$lib/api';
|
||||
import { Badge, EmptyState, Skeleton, Table } from '$lib/components/ui';
|
||||
|
||||
let agents = $state<Agent[]>([]);
|
||||
let loading = $state(true);
|
||||
let error = $state('');
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
try {
|
||||
agents = await listAgents();
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
load();
|
||||
|
||||
// A host is "stale" once it's gone quiet for longer than a few of its
|
||||
// own heartbeat intervals -- 3x, with a 5-minute floor for an agent
|
||||
// that's never reported a heartbeat interval at all (heartbeat_
|
||||
// interval_ms == 0, e.g. one built before this feature existed).
|
||||
// This is a client-side display heuristic only, distinct from and
|
||||
// looser than the real alerting mechanism -- see
|
||||
// /docs/agent-heartbeat-monitoring.md for the actual absence-alert-
|
||||
// rule-based detection this page doesn't replace.
|
||||
function isStale(a: Agent): boolean {
|
||||
const thresholdMs = Math.max(a.heartbeat_interval_ms * 3, 5 * 60 * 1000);
|
||||
return Date.now() - new Date(a.last_seen_at).getTime() > thresholdMs;
|
||||
}
|
||||
|
||||
function relativeTime(iso: string): string {
|
||||
const ms = Date.now() - new Date(iso).getTime();
|
||||
if (ms < 60_000) return `${Math.max(0, Math.round(ms / 1000))}s ago`;
|
||||
if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m ago`;
|
||||
if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h ago`;
|
||||
return `${Math.round(ms / 86_400_000)}d ago`;
|
||||
}
|
||||
</script>
|
||||
|
||||
<main>
|
||||
<h1>Agents</h1>
|
||||
<p class="subtitle">
|
||||
Linux/Windows log collection agents that have checked in at least once.
|
||||
</p>
|
||||
{#if error}<p class="error">Error: {error}</p>{/if}
|
||||
|
||||
{#if loading}
|
||||
<div class="skeleton-list">
|
||||
{#each Array(3) as _, i (i)}
|
||||
<Skeleton height="2.25rem" />
|
||||
{/each}
|
||||
</div>
|
||||
{:else if agents.length === 0}
|
||||
<EmptyState
|
||||
icon="●"
|
||||
title="No agents have checked in yet"
|
||||
description="An agent appears here automatically the first time it successfully calls in to ingest -- there's no manual registration step. See the agent README for how to point one at this deployment."
|
||||
/>
|
||||
{:else}
|
||||
<Table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Service</th>
|
||||
<th>Version</th>
|
||||
<th>Last seen</th>
|
||||
<th>Status</th>
|
||||
<th>Config</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each agents as a (a.id)}
|
||||
<tr>
|
||||
<td><a href={`/agents/${encodeURIComponent(a.host)}`}>{a.host}</a></td>
|
||||
<td>{a.service}</td>
|
||||
<td>{a.agent_version || '—'}</td>
|
||||
<td>{relativeTime(a.last_seen_at)}</td>
|
||||
<td>
|
||||
{#if isStale(a)}
|
||||
<Badge tone="danger">stale</Badge>
|
||||
{:else}
|
||||
<Badge tone="success">healthy</Badge>
|
||||
{/if}
|
||||
</td>
|
||||
<td>
|
||||
{#if a.pending}
|
||||
<Badge tone="accent">pending</Badge>
|
||||
{:else if a.desired_override}
|
||||
<Badge tone="neutral">applied</Badge>
|
||||
{/if}
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</Table>
|
||||
{/if}
|
||||
</main>
|
||||
|
||||
<style>
|
||||
main {
|
||||
max-width: 56rem;
|
||||
}
|
||||
h1 {
|
||||
font-size: var(--text-xl);
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
.subtitle {
|
||||
color: var(--color-text-muted);
|
||||
font-size: var(--text-sm);
|
||||
margin-bottom: var(--space-5);
|
||||
}
|
||||
.error {
|
||||
color: var(--color-danger);
|
||||
}
|
||||
.skeleton-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
a {
|
||||
color: var(--color-text);
|
||||
font-weight: var(--font-weight-medium);
|
||||
text-decoration: none;
|
||||
}
|
||||
a:hover {
|
||||
color: var(--color-accent);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,3 @@
|
||||
// No route params, data comes from a client-side fetch -- same shape as
|
||||
// dashboards/+page.ts.
|
||||
export const prerender = true;
|
||||
@@ -0,0 +1,237 @@
|
||||
<script lang="ts">
|
||||
import { page } from '$app/state';
|
||||
import { getAgent, setAgentConfig, clearAgentConfig, type Agent } from '$lib/api';
|
||||
import { Badge, Button, Input, Skeleton } from '$lib/components/ui';
|
||||
|
||||
const host = page.params.host!;
|
||||
|
||||
let agent = $state<Agent | null>(null);
|
||||
let loading = $state(true);
|
||||
let error = $state('');
|
||||
let saving = $state(false);
|
||||
let saveError = $state('');
|
||||
|
||||
// Editable form fields -- seeded from the agent's current
|
||||
// desired_override when one exists, otherwise from its currently-
|
||||
// reported effective values. Saving always PUTs the complete set
|
||||
// (api/agents.Store.SetOverride replaces the whole stored override,
|
||||
// it doesn't patch individual fields), so every field needs a
|
||||
// sensible starting value regardless of whether an override exists
|
||||
// yet -- see /docs/agent-management-design.md. Kept as strings since
|
||||
// the shared <Input> component's `value` prop is typed string
|
||||
// (native <input type=number>'s bound value is a string too, DOM-
|
||||
// side) -- converted to numbers only at save().
|
||||
let batchMaxSize = $state('0');
|
||||
let batchFlushIntervalMs = $state('0');
|
||||
let heartbeatEnabled = $state(true);
|
||||
let heartbeatIntervalMs = $state('0');
|
||||
let journaldUnit = $state('');
|
||||
|
||||
function resetForm(a: Agent) {
|
||||
const o = a.desired_override;
|
||||
batchMaxSize = String(o?.batch_max_size ?? a.batch_max_size);
|
||||
batchFlushIntervalMs = String(o?.batch_flush_interval_ms ?? a.batch_flush_interval_ms);
|
||||
heartbeatEnabled = o?.heartbeat_enabled ?? a.heartbeat_enabled;
|
||||
heartbeatIntervalMs = String(o?.heartbeat_interval_ms ?? a.heartbeat_interval_ms);
|
||||
journaldUnit = o?.journald_unit ?? '';
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = '';
|
||||
try {
|
||||
agent = await getAgent(host);
|
||||
resetForm(agent);
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
load();
|
||||
|
||||
async function save() {
|
||||
saving = true;
|
||||
saveError = '';
|
||||
try {
|
||||
agent = await setAgentConfig(host, {
|
||||
batch_max_size: Number(batchMaxSize),
|
||||
batch_flush_interval_ms: Number(batchFlushIntervalMs),
|
||||
heartbeat_enabled: heartbeatEnabled,
|
||||
heartbeat_interval_ms: Number(heartbeatIntervalMs),
|
||||
...(agent?.source_kind === 'journald' ? { journald_unit: journaldUnit } : {})
|
||||
});
|
||||
} catch (e) {
|
||||
saveError = e instanceof Error ? e.message : String(e);
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function revert() {
|
||||
saving = true;
|
||||
saveError = '';
|
||||
try {
|
||||
await clearAgentConfig(host);
|
||||
agent = await getAgent(host);
|
||||
resetForm(agent);
|
||||
} catch (e) {
|
||||
saveError = e instanceof Error ? e.message : String(e);
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
function relativeTime(iso: string): string {
|
||||
const ms = Date.now() - new Date(iso).getTime();
|
||||
if (ms < 60_000) return `${Math.max(0, Math.round(ms / 1000))}s ago`;
|
||||
if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m ago`;
|
||||
if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h ago`;
|
||||
return `${Math.round(ms / 86_400_000)}d ago`;
|
||||
}
|
||||
</script>
|
||||
|
||||
<main>
|
||||
<a class="back" href="/agents">← Agents</a>
|
||||
<h1>{host}</h1>
|
||||
|
||||
{#if loading}
|
||||
<Skeleton height="12rem" />
|
||||
{:else if error}
|
||||
<p class="error">Error: {error}</p>
|
||||
{:else if agent}
|
||||
<section class="reported">
|
||||
<h2>Reported</h2>
|
||||
<dl>
|
||||
<dt>Service</dt>
|
||||
<dd>{agent.service}</dd>
|
||||
<dt>Version</dt>
|
||||
<dd>{agent.agent_version || '—'}</dd>
|
||||
<dt>Source</dt>
|
||||
<dd>{agent.source_kind}{agent.source_detail ? ` (${agent.source_detail})` : ''}</dd>
|
||||
<dt>First seen</dt>
|
||||
<dd>{relativeTime(agent.first_seen_at)}</dd>
|
||||
<dt>Last seen</dt>
|
||||
<dd>{relativeTime(agent.last_seen_at)}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<section class="config">
|
||||
<h2>
|
||||
Remote config
|
||||
{#if agent.pending}
|
||||
<Badge tone="accent">pending — agent hasn't applied this yet</Badge>
|
||||
{:else if agent.desired_override}
|
||||
<Badge tone="neutral">applied</Badge>
|
||||
{/if}
|
||||
</h2>
|
||||
<p class="hint">
|
||||
Changes here don't touch the agent's local config file -- they're an override the agent fetches and applies
|
||||
on its own schedule (its heartbeat interval), and revert automatically if the agent restarts before its next
|
||||
check-in re-syncs them. Connection details (TLS, ingest endpoint) are never remotely editable.
|
||||
</p>
|
||||
|
||||
<div class="field">
|
||||
<label for="batch-max-size">Batch max size</label>
|
||||
<Input id="batch-max-size" type="number" min="1" bind:value={batchMaxSize} />
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="batch-flush-ms">Batch flush interval (ms)</label>
|
||||
<Input id="batch-flush-ms" type="number" min="100" bind:value={batchFlushIntervalMs} />
|
||||
</div>
|
||||
<div class="field checkbox">
|
||||
<label><input type="checkbox" bind:checked={heartbeatEnabled} /> Heartbeat enabled</label>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="heartbeat-ms">Heartbeat interval (ms)</label>
|
||||
<Input id="heartbeat-ms" type="number" min="5000" bind:value={heartbeatIntervalMs} />
|
||||
</div>
|
||||
{#if agent.source_kind === 'journald'}
|
||||
<div class="field">
|
||||
<label for="journald-unit">Journald unit filter</label>
|
||||
<Input id="journald-unit" placeholder="(empty = whole journal)" bind:value={journaldUnit} />
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if saveError}<p class="error">Error: {saveError}</p>{/if}
|
||||
|
||||
<div class="actions">
|
||||
<Button variant="primary" onclick={save} disabled={saving}>Save</Button>
|
||||
{#if agent.desired_override}
|
||||
<Button variant="secondary" onclick={revert} disabled={saving}>Revert to local config</Button>
|
||||
{/if}
|
||||
</div>
|
||||
</section>
|
||||
{/if}
|
||||
</main>
|
||||
|
||||
<style>
|
||||
main {
|
||||
max-width: 40rem;
|
||||
}
|
||||
.back {
|
||||
font-size: var(--text-sm);
|
||||
color: var(--color-text-muted);
|
||||
text-decoration: none;
|
||||
}
|
||||
.back:hover {
|
||||
color: var(--color-accent);
|
||||
}
|
||||
h1 {
|
||||
font-size: var(--text-xl);
|
||||
margin: var(--space-2) 0 var(--space-5);
|
||||
font-family: var(--font-mono);
|
||||
}
|
||||
h2 {
|
||||
font-size: var(--text-base);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.error {
|
||||
color: var(--color-danger);
|
||||
}
|
||||
.reported {
|
||||
margin-bottom: var(--space-6);
|
||||
}
|
||||
dl {
|
||||
display: grid;
|
||||
grid-template-columns: auto 1fr;
|
||||
gap: var(--space-1) var(--space-4);
|
||||
font-size: var(--text-sm);
|
||||
}
|
||||
dt {
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
dd {
|
||||
margin: 0;
|
||||
}
|
||||
.hint {
|
||||
color: var(--color-text-muted);
|
||||
font-size: var(--text-sm);
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
.field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-1);
|
||||
margin-bottom: var(--space-3);
|
||||
max-width: 20rem;
|
||||
}
|
||||
.field label {
|
||||
font-size: var(--text-sm);
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
.field.checkbox label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
color: var(--color-text);
|
||||
}
|
||||
.actions {
|
||||
display: flex;
|
||||
gap: var(--space-3);
|
||||
margin-top: var(--space-5);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,4 @@
|
||||
// The host param doesn't exist at build time -- same reasoning as
|
||||
// dashboards/[id]/+page.ts.
|
||||
export const prerender = false;
|
||||
export const ssr = false;
|
||||
Reference in New Issue
Block a user