Closes the threat model's headline finding for the SQL query path:
enterprise/internal/tenantprovision does real CREATE DATABASE/USER/GRANT
against ClickHouse, and enterprise/internal/chrunner is a per-tenant
connection registry implementing api's SQLRunner interface, resolving
the tenant from the authenticated request identity -- never a
caller-suppliable parameter. Both are wired into a new binary,
enterprise/cmd/enterprise-api, alongside the unchanged single-tenant
api/cmd/api, since AGPL core can never import enterprise/ and Go's own
internal/ package visibility rules meant enterprise/ couldn't implement
core's SQLRunner interface without importing the package that defines
it. That required moving api/internal/{authz,queryapi,dashboards,
querylang/executor,searchclient,httpserver} out of internal/ -- the
minimal set enterprise-api needs to import; querylang's compiler
internals (planner/lexer/parser/ast/ir) and api's own config stay
internal, since nothing outside api needs them directly.
Also finally wires enterprise/internal/audit into queryapi.AuditLogger
(nil since Phase 4 task 4) via a new adapter, and adds live-ClickHouse
integration tests for two of the four adversarial probes named in
docs/phase-4-isolation-design.md's verification plan.
Corrected several overclaims in the docs while writing this up: an
earlier claim that rbacstore's CRUD was "verified against a live
Postgres" was never actually true in this environment (only
internal/audit was, earlier in this phase, before Docker access was
lost) -- threat-model.md, phase-4-runbook.md, CLAUDE.md, and
enterprise/README.md all now distinguish "a real integration test
exists" from "this was confirmed against a live database."
Still not built: Tantivy/free-text tenant isolation
(enterprise/internal/searchclient), and any deployment-topology
mechanism that actually routes traffic to enterprise-api instead of
plain api -- both binaries exist side by side today with nothing
enforcing or flagging which one a deployment runs.
186 lines
6.7 KiB
Go
186 lines
6.7 KiB
Go
// Integration tests against a real ClickHouse -- exercises the actual
|
|
// question this package exists to answer: does a query authenticated as
|
|
// tenant A ever see tenant B's data. Uses enterprise/internal/
|
|
// tenantprovision to set up real per-tenant users first (this is the
|
|
// adversarial probe api/queryapi/tenant_isolation_gap_test.go's
|
|
// TestAdversarial_ClickHouseUserCannotReadOtherTenantDatabaseByFullyQualifiedName
|
|
// names as blocked -- this is where it stops being blocked, at the
|
|
// chrunner/query-execution layer specifically, complementing
|
|
// tenantprovision's own version of the same probe at the raw-SQL-user
|
|
// layer).
|
|
//
|
|
// Skipped unless CHRUNNER_TEST_CLICKHOUSE_ADDR is set; run via:
|
|
//
|
|
// docker run --rm --network sentry_default -v $(pwd)/../../..:/src -w /src/enterprise \
|
|
// -e CHRUNNER_TEST_CLICKHOUSE_ADDR=clickhouse:9000 \
|
|
// -e CHRUNNER_TEST_CLICKHOUSE_PASSWORD=sentry-dev-only \
|
|
// golang:1.25-alpine go test ./internal/chrunner/... -v
|
|
package chrunner
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
chdriver "github.com/ClickHouse/clickhouse-go/v2"
|
|
"github.com/google/uuid"
|
|
"github.com/sentry/sentry/api/authz"
|
|
|
|
"github.com/sentry/sentry/enterprise/internal/tenantprovision"
|
|
)
|
|
|
|
func testAddr(t *testing.T) string {
|
|
t.Helper()
|
|
addr := os.Getenv("CHRUNNER_TEST_CLICKHOUSE_ADDR")
|
|
if addr == "" {
|
|
t.Skip("CHRUNNER_TEST_CLICKHOUSE_ADDR not set -- skipping live-ClickHouse integration test")
|
|
}
|
|
return addr
|
|
}
|
|
|
|
func provisionTestTenant(t *testing.T, addr string) (tenantID string, creds tenantprovision.Credentials) {
|
|
t.Helper()
|
|
admin, err := chdriver.Open(&chdriver.Options{
|
|
Addr: []string{addr},
|
|
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("opening admin connection: %v", err)
|
|
}
|
|
t.Cleanup(func() { admin.Close() })
|
|
|
|
tenantID = "cr" + uuid.NewString()[:8]
|
|
creds, err = tenantprovision.New(admin).ProvisionClickHouse(context.Background(), tenantID)
|
|
if err != nil {
|
|
t.Fatalf("provisioning tenant %s: %v", tenantID, err)
|
|
}
|
|
return tenantID, creds
|
|
}
|
|
|
|
func TestRegistryRoutesQueryToCorrectTenant(t *testing.T) {
|
|
addr := testAddr(t)
|
|
ctx := context.Background()
|
|
tenantA, credsA := provisionTestTenant(t, addr)
|
|
|
|
// Seed a distinguishing row directly as the tenant (SELECT-only
|
|
// grant means chrunner's own connection can't INSERT -- use a
|
|
// throwaway admin connection to seed data, matching how a real
|
|
// deployment's ingest path would write, not how api's read-only
|
|
// query path does).
|
|
admin, err := chdriver.Open(&chdriver.Options{
|
|
Addr: []string{addr},
|
|
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("opening admin connection: %v", err)
|
|
}
|
|
defer admin.Close()
|
|
if err := admin.Exec(ctx, fmt.Sprintf("CREATE TABLE `%s`.marker (id UInt8) ENGINE = Memory", tenantA)); err != nil {
|
|
t.Fatalf("creating marker table: %v", err)
|
|
}
|
|
if err := admin.Exec(ctx, fmt.Sprintf("INSERT INTO `%s`.marker VALUES (42)", tenantA)); err != nil {
|
|
t.Fatalf("seeding marker row: %v", err)
|
|
}
|
|
|
|
reg, err := New(ctx, addr, []DataSource{
|
|
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("New: %v", err)
|
|
}
|
|
defer reg.Close()
|
|
|
|
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: tenantA, Role: authz.RoleViewer})
|
|
result, err := reg.RunSQL(reqCtx, "SELECT id FROM marker")
|
|
if err != nil {
|
|
t.Fatalf("RunSQL: %v", err)
|
|
}
|
|
if len(result.Rows) != 1 || result.Rows[0][0] != uint8(42) {
|
|
t.Fatalf("unexpected result: %+v", result.Rows)
|
|
}
|
|
}
|
|
|
|
func TestRegistryRefusesQueryWithNoTenantContext(t *testing.T) {
|
|
addr := testAddr(t)
|
|
ctx := context.Background()
|
|
tenantA, credsA := provisionTestTenant(t, addr)
|
|
|
|
reg, err := New(ctx, addr, []DataSource{
|
|
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("New: %v", err)
|
|
}
|
|
defer reg.Close()
|
|
|
|
if _, err := reg.RunSQL(ctx, "SELECT 1"); err == nil {
|
|
t.Fatal("expected RunSQL to refuse a request with no authenticated identity in context")
|
|
}
|
|
}
|
|
|
|
func TestRegistryRefusesUnknownTenant(t *testing.T) {
|
|
addr := testAddr(t)
|
|
ctx := context.Background()
|
|
tenantA, credsA := provisionTestTenant(t, addr)
|
|
|
|
reg, err := New(ctx, addr, []DataSource{
|
|
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("New: %v", err)
|
|
}
|
|
defer reg.Close()
|
|
|
|
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: "some-other-tenant-never-provisioned", Role: authz.RoleViewer})
|
|
if _, err := reg.RunSQL(reqCtx, "SELECT 1"); err == nil {
|
|
t.Fatal("expected RunSQL to refuse a tenant with no provisioned connection, not silently fall back")
|
|
}
|
|
}
|
|
|
|
// TestRegistryTenantCannotReadOtherTenantEvenViaRawSQL is the full
|
|
// end-to-end adversarial probe: two tenants, two connections inside one
|
|
// Registry, and a raw-SQL attempt (which the query language's escape
|
|
// hatch would pass straight through unmodified) to read the other
|
|
// tenant's data by fully-qualified name. This is what proves the
|
|
// connection-layer isolation model actually holds through chrunner, not
|
|
// just through tenantprovision's own grants (already covered by
|
|
// tenantprovision_test.go) -- this test exercises the exact code path
|
|
// api/queryapi.Handler calls in production.
|
|
func TestRegistryTenantCannotReadOtherTenantEvenViaRawSQL(t *testing.T) {
|
|
addr := testAddr(t)
|
|
ctx := context.Background()
|
|
tenantA, credsA := provisionTestTenant(t, addr)
|
|
tenantB, credsB := provisionTestTenant(t, addr)
|
|
|
|
admin, err := chdriver.Open(&chdriver.Options{
|
|
Addr: []string{addr},
|
|
Auth: chdriver.Auth{Database: "default", Username: "default", Password: os.Getenv("CHRUNNER_TEST_CLICKHOUSE_PASSWORD")},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("opening admin connection: %v", err)
|
|
}
|
|
defer admin.Close()
|
|
if err := admin.Exec(ctx, fmt.Sprintf("CREATE TABLE `%s`.secret (id UInt8) ENGINE = Memory", tenantB)); err != nil {
|
|
t.Fatalf("creating secret table: %v", err)
|
|
}
|
|
if err := admin.Exec(ctx, fmt.Sprintf("INSERT INTO `%s`.secret VALUES (99)", tenantB)); err != nil {
|
|
t.Fatalf("seeding secret row: %v", err)
|
|
}
|
|
|
|
reg, err := New(ctx, addr, []DataSource{
|
|
{TenantID: tenantA, Database: tenantA, Username: credsA.Username, Password: credsA.Password},
|
|
{TenantID: tenantB, Database: tenantB, Username: credsB.Username, Password: credsB.Password},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("New: %v", err)
|
|
}
|
|
defer reg.Close()
|
|
|
|
reqCtx := authz.WithIdentity(ctx, authz.Identity{TenantID: tenantA, Role: authz.RoleViewer})
|
|
_, err = reg.RunSQL(reqCtx, fmt.Sprintf("SELECT * FROM `%s`.secret", tenantB))
|
|
if err == nil {
|
|
t.Fatal("tenant A's request was able to read tenant B's database by fully-qualified name -- isolation is broken")
|
|
}
|
|
}
|