Scaffold Phase 0: agent -> Redpanda -> ingest -> ClickHouse -> api -> web

End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
2026-08-13 08:25:19 -07:00
commit b6b092c912
92 changed files with 7796 additions and 0 deletions
@@ -0,0 +1,376 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.12
+// 	protoc        v7.35.1
+// source: sentry/logs/v1/logs.proto
+
+package logsv1
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// Severity follows OTel's severity number ranges (1-24), collapsed here to
+// the coarse names agents actually need to set. Numeric value stored
+// downstream may be a full OTel severity_number computed by ingest.
+type Severity int32
+
+const (
+	Severity_SEVERITY_UNSPECIFIED Severity = 0
+	Severity_SEVERITY_TRACE       Severity = 1
+	Severity_SEVERITY_DEBUG       Severity = 2
+	Severity_SEVERITY_INFO        Severity = 3
+	Severity_SEVERITY_WARN        Severity = 4
+	Severity_SEVERITY_ERROR       Severity = 5
+	Severity_SEVERITY_FATAL       Severity = 6
+)
+
+// Enum value maps for Severity.
+var (
+	Severity_name = map[int32]string{
+		0: "SEVERITY_UNSPECIFIED",
+		1: "SEVERITY_TRACE",
+		2: "SEVERITY_DEBUG",
+		3: "SEVERITY_INFO",
+		4: "SEVERITY_WARN",
+		5: "SEVERITY_ERROR",
+		6: "SEVERITY_FATAL",
+	}
+	Severity_value = map[string]int32{
+		"SEVERITY_UNSPECIFIED": 0,
+		"SEVERITY_TRACE":       1,
+		"SEVERITY_DEBUG":       2,
+		"SEVERITY_INFO":        3,
+		"SEVERITY_WARN":        4,
+		"SEVERITY_ERROR":       5,
+		"SEVERITY_FATAL":       6,
+	}
+)
+
+func (x Severity) Enum() *Severity {
+	p := new(Severity)
+	*p = x
+	return p
+}
+
+func (x Severity) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Severity) Descriptor() protoreflect.EnumDescriptor {
+	return file_sentry_logs_v1_logs_proto_enumTypes[0].Descriptor()
+}
+
+func (Severity) Type() protoreflect.EnumType {
+	return &file_sentry_logs_v1_logs_proto_enumTypes[0]
+}
+
+func (x Severity) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Severity.Descriptor instead.
+func (Severity) EnumDescriptor() ([]byte, []int) {
+	return file_sentry_logs_v1_logs_proto_rawDescGZIP(), []int{0}
+}
+
+type LogRecord struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Unix epoch nanoseconds, set by the agent at time of read (not parse or
+	// send time) to preserve original ordering as closely as possible.
+	TimestampUnixNano int64 `protobuf:"varint,1,opt,name=timestamp_unix_nano,json=timestampUnixNano,proto3" json:"timestamp_unix_nano,omitempty"`
+	// Hostname the agent is running on. Agent fills this from its own config
+	// or system hostname; not trusted as an identity claim (mTLS client cert
+	// is the identity boundary).
+	Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"`
+	// Logical service/unit name. For journald sources, this is typically the
+	// systemd unit name; for file sources, it comes from agent config.
+	Service  string   `protobuf:"bytes,3,opt,name=service,proto3" json:"service,omitempty"`
+	Severity Severity `protobuf:"varint,4,opt,name=severity,proto3,enum=sentry.logs.v1.Severity" json:"severity,omitempty"`
+	// Original, unparsed log line. Always populated, even when structured
+	// fields below are also present, per the schema-on-read fallback
+	// requirement in CLAUDE.md.
+	Message string `protobuf:"bytes,5,opt,name=message,proto3" json:"message,omitempty"`
+	// Structured fields extracted by the agent's parser (e.g. RFC 5424
+	// syslog header fields). Empty when the raw-passthrough fallback fires.
+	Attributes    map[string]string `protobuf:"bytes,6,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *LogRecord) Reset() {
+	*x = LogRecord{}
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *LogRecord) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LogRecord) ProtoMessage() {}
+
+func (x *LogRecord) ProtoReflect() protoreflect.Message {
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LogRecord.ProtoReflect.Descriptor instead.
+func (*LogRecord) Descriptor() ([]byte, []int) {
+	return file_sentry_logs_v1_logs_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *LogRecord) GetTimestampUnixNano() int64 {
+	if x != nil {
+		return x.TimestampUnixNano
+	}
+	return 0
+}
+
+func (x *LogRecord) GetHost() string {
+	if x != nil {
+		return x.Host
+	}
+	return ""
+}
+
+func (x *LogRecord) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
+}
+
+func (x *LogRecord) GetSeverity() Severity {
+	if x != nil {
+		return x.Severity
+	}
+	return Severity_SEVERITY_UNSPECIFIED
+}
+
+func (x *LogRecord) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *LogRecord) GetAttributes() map[string]string {
+	if x != nil {
+		return x.Attributes
+	}
+	return nil
+}
+
+type PushBatchRequest struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Agent-assigned identifier for dedup/idempotency on retry. Ingest may
+	// use this to avoid double-writing a batch if a retry follows a
+	// timeout on an actually-successful push.
+	BatchId       string       `protobuf:"bytes,1,opt,name=batch_id,json=batchId,proto3" json:"batch_id,omitempty"`
+	Records       []*LogRecord `protobuf:"bytes,2,rep,name=records,proto3" json:"records,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PushBatchRequest) Reset() {
+	*x = PushBatchRequest{}
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PushBatchRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PushBatchRequest) ProtoMessage() {}
+
+func (x *PushBatchRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PushBatchRequest.ProtoReflect.Descriptor instead.
+func (*PushBatchRequest) Descriptor() ([]byte, []int) {
+	return file_sentry_logs_v1_logs_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *PushBatchRequest) GetBatchId() string {
+	if x != nil {
+		return x.BatchId
+	}
+	return ""
+}
+
+func (x *PushBatchRequest) GetRecords() []*LogRecord {
+	if x != nil {
+		return x.Records
+	}
+	return nil
+}
+
+type PushBatchResponse struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Number of records ingest accepted. Phase 0: batches are all-or-nothing,
+	// so this equals len(records) on success. Partial-acceptance semantics
+	// are not implemented yet.
+	Accepted      uint32 `protobuf:"varint,1,opt,name=accepted,proto3" json:"accepted,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PushBatchResponse) Reset() {
+	*x = PushBatchResponse{}
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PushBatchResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PushBatchResponse) ProtoMessage() {}
+
+func (x *PushBatchResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_sentry_logs_v1_logs_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PushBatchResponse.ProtoReflect.Descriptor instead.
+func (*PushBatchResponse) Descriptor() ([]byte, []int) {
+	return file_sentry_logs_v1_logs_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *PushBatchResponse) GetAccepted() uint32 {
+	if x != nil {
+		return x.Accepted
+	}
+	return 0
+}
+
+var File_sentry_logs_v1_logs_proto protoreflect.FileDescriptor
+
+const file_sentry_logs_v1_logs_proto_rawDesc = "" +
+	"\n" +
+	"\x19sentry/logs/v1/logs.proto\x12\x0esentry.logs.v1\"\xc3\x02\n" +
+	"\tLogRecord\x12.\n" +
+	"\x13timestamp_unix_nano\x18\x01 \x01(\x03R\x11timestampUnixNano\x12\x12\n" +
+	"\x04host\x18\x02 \x01(\tR\x04host\x12\x18\n" +
+	"\aservice\x18\x03 \x01(\tR\aservice\x124\n" +
+	"\bseverity\x18\x04 \x01(\x0e2\x18.sentry.logs.v1.SeverityR\bseverity\x12\x18\n" +
+	"\amessage\x18\x05 \x01(\tR\amessage\x12I\n" +
+	"\n" +
+	"attributes\x18\x06 \x03(\v2).sentry.logs.v1.LogRecord.AttributesEntryR\n" +
+	"attributes\x1a=\n" +
+	"\x0fAttributesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"b\n" +
+	"\x10PushBatchRequest\x12\x19\n" +
+	"\bbatch_id\x18\x01 \x01(\tR\abatchId\x123\n" +
+	"\arecords\x18\x02 \x03(\v2\x19.sentry.logs.v1.LogRecordR\arecords\"/\n" +
+	"\x11PushBatchResponse\x12\x1a\n" +
+	"\baccepted\x18\x01 \x01(\rR\baccepted*\x9a\x01\n" +
+	"\bSeverity\x12\x18\n" +
+	"\x14SEVERITY_UNSPECIFIED\x10\x00\x12\x12\n" +
+	"\x0eSEVERITY_TRACE\x10\x01\x12\x12\n" +
+	"\x0eSEVERITY_DEBUG\x10\x02\x12\x11\n" +
+	"\rSEVERITY_INFO\x10\x03\x12\x11\n" +
+	"\rSEVERITY_WARN\x10\x04\x12\x12\n" +
+	"\x0eSEVERITY_ERROR\x10\x05\x12\x12\n" +
+	"\x0eSEVERITY_FATAL\x10\x062]\n" +
+	"\tLogIngest\x12P\n" +
+	"\tPushBatch\x12 .sentry.logs.v1.PushBatchRequest\x1a!.sentry.logs.v1.PushBatchResponseB6Z4github.com/sentry/sentry/proto/sentry/logs/v1;logsv1b\x06proto3"
+
+var (
+	file_sentry_logs_v1_logs_proto_rawDescOnce sync.Once
+	file_sentry_logs_v1_logs_proto_rawDescData []byte
+)
+
+func file_sentry_logs_v1_logs_proto_rawDescGZIP() []byte {
+	file_sentry_logs_v1_logs_proto_rawDescOnce.Do(func() {
+		file_sentry_logs_v1_logs_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sentry_logs_v1_logs_proto_rawDesc), len(file_sentry_logs_v1_logs_proto_rawDesc)))
+	})
+	return file_sentry_logs_v1_logs_proto_rawDescData
+}
+
+var file_sentry_logs_v1_logs_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_sentry_logs_v1_logs_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_sentry_logs_v1_logs_proto_goTypes = []any{
+	(Severity)(0),             // 0: sentry.logs.v1.Severity
+	(*LogRecord)(nil),         // 1: sentry.logs.v1.LogRecord
+	(*PushBatchRequest)(nil),  // 2: sentry.logs.v1.PushBatchRequest
+	(*PushBatchResponse)(nil), // 3: sentry.logs.v1.PushBatchResponse
+	nil,                       // 4: sentry.logs.v1.LogRecord.AttributesEntry
+}
+var file_sentry_logs_v1_logs_proto_depIdxs = []int32{
+	0, // 0: sentry.logs.v1.LogRecord.severity:type_name -> sentry.logs.v1.Severity
+	4, // 1: sentry.logs.v1.LogRecord.attributes:type_name -> sentry.logs.v1.LogRecord.AttributesEntry
+	1, // 2: sentry.logs.v1.PushBatchRequest.records:type_name -> sentry.logs.v1.LogRecord
+	2, // 3: sentry.logs.v1.LogIngest.PushBatch:input_type -> sentry.logs.v1.PushBatchRequest
+	3, // 4: sentry.logs.v1.LogIngest.PushBatch:output_type -> sentry.logs.v1.PushBatchResponse
+	4, // [4:5] is the sub-list for method output_type
+	3, // [3:4] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_sentry_logs_v1_logs_proto_init() }
+func file_sentry_logs_v1_logs_proto_init() {
+	if File_sentry_logs_v1_logs_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_sentry_logs_v1_logs_proto_rawDesc), len(file_sentry_logs_v1_logs_proto_rawDesc)),
+			NumEnums:      1,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_sentry_logs_v1_logs_proto_goTypes,
+		DependencyIndexes: file_sentry_logs_v1_logs_proto_depIdxs,
+		EnumInfos:         file_sentry_logs_v1_logs_proto_enumTypes,
+		MessageInfos:      file_sentry_logs_v1_logs_proto_msgTypes,
+	}.Build()
+	File_sentry_logs_v1_logs_proto = out.File
+	file_sentry_logs_v1_logs_proto_goTypes = nil
+	file_sentry_logs_v1_logs_proto_depIdxs = nil
+}