Add agent inventory, management, and remote config

Extends the heartbeat mechanism with a second gRPC service on the same mTLS channel (AgentControl.CheckIn, agent-initiated on the existing heartbeat ticker -- still push-only, no inbound port on any agent) so an agent reports its running config and can pick up an operator-set override. A new web UI section (/agents) lists every agent that's checked in, shows its reported config, and lets an operator edit a narrow, deliberately-scoped subset remotely: batch/heartbeat tuning, and (journald sources only) the unit filter. TLS material and the ingest endpoint are never reportable or remotely editable, by proto shape rather than a validation rule -- a bad or malicious edit there could permanently strand an agent or redirect where its logs go, unlike every other editable field, which only degrades behavior. An override lives only in the agent's memory (agent.toml is never rewritten) and re-syncs on the agent's own schedule; changing the journald filter aborts and respawns the source task since there's no other way to change what's being tailed. Building the hot-reload path surfaced a real, independent, pre-existing bug: shutdown was using poll_timeout(), which only drains once flush_interval has elapsed, silently dropping anything buffered more recently on every graceful shutdown that landed between flushes -- fixed with a new unconditional Batcher::flush_all(), now used at both shutdown and hot-reload. Verified live end-to-end against a real stack: an edited heartbeat interval changed a running agent's actual send cadence within one check-in cycle (confirmed by the real timestamps landing in ClickHouse), and an edited journald filter triggered a real source restart, both reflected back in the next reported-config snapshot. See /docs/agent-management-design.md.
2026-08-16 18:08:51 -07:00
parent 4df6931869
commit 4f0da1ae5e
29 changed files with 2618 additions and 53 deletions
@@ -0,0 +1,108 @@
+// Package agentregistry is the Postgres-backed implementation of
+// grpcserver.AgentRegistry -- ingest's half of agent inventory/remote
+// config (see /docs/agent-management-design.md). Writes into the same
+// sentry_metadata Postgres api reads/writes from for the web UI's
+// inventory and edit-config views (api/agents), the same shared-schema-
+// different-services shape alerting and api already use for dashboards/
+// alert_rules.
+package agentregistry
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"github.com/sentry/sentry/ingest/internal/grpcserver"
+)
+
+// defaultTenantID is used when no TenantResolver is configured (empty
+// tenantID from grpcserver) -- the same 'default' tenant row every
+// single-tenant deployment's Postgres already has, seeded by
+// metadata/migrations/0019_seed_default_tenant.sql.
+const defaultTenantID = "default"
+
+// overrideFields is the JSON shape stored in agents.desired_override.
+// Deliberately duplicated in api/agents rather than imported -- these
+// are two different Go modules, and this codebase's established
+// convention (see enterprise/internal/apiconfig.AIConfig,
+// grpcserver.TenantIDHeaderKey) is to duplicate a small shared shape
+// across a module boundary rather than couple two independently
+// deployable services' builds together. Keep the two in sync by hand.
+type overrideFields struct {
+	BatchMaxSize         *uint64 `json:"batch_max_size,omitempty"`
+	BatchFlushIntervalMS *uint64 `json:"batch_flush_interval_ms,omitempty"`
+	HeartbeatEnabled     *bool   `json:"heartbeat_enabled,omitempty"`
+	HeartbeatIntervalMS  *uint64 `json:"heartbeat_interval_ms,omitempty"`
+	JournaldUnit         *string `json:"journald_unit,omitempty"`
+}
+
+type Registry struct {
+	pool *pgxpool.Pool
+}
+
+func New(pool *pgxpool.Pool) *Registry {
+	return &Registry{pool: pool}
+}
+
+var _ grpcserver.AgentRegistry = (*Registry)(nil)
+
+func (r *Registry) CheckIn(ctx context.Context, tenantID string, info grpcserver.AgentCheckIn) (grpcserver.AgentOverride, error) {
+	if tenantID == "" {
+		tenantID = defaultTenantID
+	}
+
+	var (
+		desiredOverride []byte
+		desiredVersion  *string
+	)
+	err := r.pool.QueryRow(ctx, `
+		INSERT INTO agents (
+			id, tenant_id, host, service,
+			reported_agent_version, reported_source_kind, reported_source_detail,
+			reported_batch_max_size, reported_batch_flush_ms,
+			reported_heartbeat_on, reported_heartbeat_ms,
+			first_seen_at, last_seen_at, applied_override_version
+		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11, now(), now(), $12)
+		ON CONFLICT (tenant_id, host) DO UPDATE SET
+			service                  = EXCLUDED.service,
+			reported_agent_version   = EXCLUDED.reported_agent_version,
+			reported_source_kind     = EXCLUDED.reported_source_kind,
+			reported_source_detail   = EXCLUDED.reported_source_detail,
+			reported_batch_max_size  = EXCLUDED.reported_batch_max_size,
+			reported_batch_flush_ms  = EXCLUDED.reported_batch_flush_ms,
+			reported_heartbeat_on    = EXCLUDED.reported_heartbeat_on,
+			reported_heartbeat_ms    = EXCLUDED.reported_heartbeat_ms,
+			last_seen_at             = now(),
+			applied_override_version = EXCLUDED.applied_override_version
+		RETURNING desired_override, desired_override_version`,
+		uuid.NewString(), tenantID, info.Host, info.Service,
+		info.AgentVersion, info.SourceKind, info.SourceDetail,
+		info.BatchMaxSize, info.BatchFlushIntervalMS,
+		info.HeartbeatEnabled, info.HeartbeatIntervalMS,
+		info.AppliedOverrideVersion,
+	).Scan(&desiredOverride, &desiredVersion)
+	if err != nil {
+		return grpcserver.AgentOverride{}, fmt.Errorf("agentregistry: upserting check-in: %w", err)
+	}
+
+	if desiredVersion == nil || len(desiredOverride) == 0 {
+		return grpcserver.AgentOverride{HasOverride: false}, nil
+	}
+
+	var fields overrideFields
+	if err := json.Unmarshal(desiredOverride, &fields); err != nil {
+		return grpcserver.AgentOverride{}, fmt.Errorf("agentregistry: parsing stored override: %w", err)
+	}
+	return grpcserver.AgentOverride{
+		HasOverride:          true,
+		BatchMaxSize:         fields.BatchMaxSize,
+		BatchFlushIntervalMS: fields.BatchFlushIntervalMS,
+		HeartbeatEnabled:     fields.HeartbeatEnabled,
+		HeartbeatIntervalMS:  fields.HeartbeatIntervalMS,
+		JournaldUnit:         fields.JournaldUnit,
+		Version:              *desiredVersion,
+	}, nil
+}