jcoffey-dev
b6b092c912
End-to-end log pipeline for Linux hosts, per /docs/architecture.md: - proto: shared gRPC contract (agent <-> ingest), Go bindings checked in - agent: Rust, musl-targeted, journald/file sourcing, RFC5424 parser, mTLS gRPC client, no required config for the common case - ingest: Go, single binary with --mode server|consumer|all; gRPC front end forwards to Redpanda unchanged, consumer normalizes and batch-writes to ClickHouse with at-least-once delivery - storage: ClickHouse schema + a plain SQL-file migration runner - api: minimal SELECT-only query endpoint, plain REST (not gRPC+gateway yet -- see api/README.md) - web: SvelteKit static SPA, one query page - transport: Redpanda compose + topic provisioning - cli: sentryctl ping stub - hack/dev-certs: throwaway CA + cert generation for local mTLS - root docker-compose.yml + docs/phase-0-runbook.md tie it together Not yet run end-to-end against real Docker/ClickHouse/Redpanda -- see the runbook's caveats section before relying on this working as-is.
96 lines
2.4 KiB
Go
96 lines
2.4 KiB
Go
// Package config loads ingest's configuration from environment variables.
|
|
// Phase 0 deliberately has no config file format of its own — env vars are
|
|
// enough for a docker-compose/k8s deployment and avoid pulling in a config
|
|
// library.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type Config struct {
|
|
GRPC GRPCConfig
|
|
TLS TLSConfig
|
|
Redpanda RedpandaConfig
|
|
ClickHouse ClickHouseConfig
|
|
Batch BatchConfig
|
|
}
|
|
|
|
type GRPCConfig struct {
|
|
ListenAddr string
|
|
}
|
|
|
|
// TLSConfig is the server-side mTLS material: the ingest service's own
|
|
// cert/key, and the CA used to verify agent client certs.
|
|
type TLSConfig struct {
|
|
CertFile string
|
|
KeyFile string
|
|
ClientCAFile string
|
|
}
|
|
|
|
type RedpandaConfig struct {
|
|
Brokers []string
|
|
Topic string
|
|
ConsumerGroup string
|
|
}
|
|
|
|
type ClickHouseConfig struct {
|
|
Addr string
|
|
Database string
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
type BatchConfig struct {
|
|
MaxSize int
|
|
FlushIntervalMS int
|
|
}
|
|
|
|
func Load() (Config, error) {
|
|
cfg := Config{
|
|
GRPC: GRPCConfig{
|
|
ListenAddr: getenv("GRPC_LISTEN_ADDR", ":4317"),
|
|
},
|
|
TLS: TLSConfig{
|
|
CertFile: getenv("TLS_CERT_FILE", "/etc/sentry-ingest/server.pem"),
|
|
KeyFile: getenv("TLS_KEY_FILE", "/etc/sentry-ingest/server-key.pem"),
|
|
ClientCAFile: getenv("TLS_CLIENT_CA_FILE", "/etc/sentry-ingest/ca.pem"),
|
|
},
|
|
Redpanda: RedpandaConfig{
|
|
Brokers: strings.Split(getenv("REDPANDA_BROKERS", "localhost:9092"), ","),
|
|
Topic: getenv("REDPANDA_TOPIC", "sentry.logs.raw"),
|
|
ConsumerGroup: getenv("REDPANDA_CONSUMER_GROUP", "sentry-ingest"),
|
|
},
|
|
ClickHouse: ClickHouseConfig{
|
|
Addr: getenv("CLICKHOUSE_ADDR", "localhost:9000"),
|
|
Database: getenv("CLICKHOUSE_DATABASE", "sentry"),
|
|
Username: getenv("CLICKHOUSE_USERNAME", "default"),
|
|
Password: getenv("CLICKHOUSE_PASSWORD", ""),
|
|
},
|
|
}
|
|
|
|
maxSize, err := strconv.Atoi(getenv("CONSUMER_BATCH_MAX_SIZE", "500"))
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("CONSUMER_BATCH_MAX_SIZE: %w", err)
|
|
}
|
|
cfg.Batch.MaxSize = maxSize
|
|
|
|
flushMS, err := strconv.Atoi(getenv("CONSUMER_BATCH_FLUSH_INTERVAL_MS", "2000"))
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("CONSUMER_BATCH_FLUSH_INTERVAL_MS: %w", err)
|
|
}
|
|
cfg.Batch.FlushIntervalMS = flushMS
|
|
|
|
return cfg, nil
|
|
}
|
|
|
|
func getenv(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|