diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 83ad123..d51ed9c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -204,10 +204,32 @@ pre-migration instance is still up, rather than after it isn't. production instance, `migrate_v016.py` migrated 219 of 12,401 settings — 1.8% — leaving 12,182 for the operator to recreate by hand, including `server.listener`. A migrated instance therefore serves nothing until - somebody rebuilds its listeners, whatever else went right. Something has - to generate that plan; the only question is whether it is this tool, - reviewably, or a human under time pressure during a cutover window. - `unmigrated.txt` (§4.9) is the input it should be built from. + somebody rebuilds its listeners, whatever else went right. + + **Status: started** (`internal/applyplan`), generating `NetworkListener` + objects from `server.listener.*` and reporting its own coverage. Listeners + came first because every other unmigrated setting degrades the server + while this one stops it being a server at all. + + Two rules the package holds to: + + - **Only mappings confirmed against a real v0.16 binary go in.** The + published schema reference gives `NetworkListener.bind` as a JSON + array; 0.16.14 rejects that. The encoding it accepts — a value-keyed + set, `{"[::]:25": true}` — was found by applying a plan to a live + recovery-mode instance and reading it back with `stalwart-cli + snapshot`. An unverified guess here produces a plan that fails at apply + time or, worse, quietly configures the wrong thing. + - **Coverage is reported, never implied.** Against the smoke instance the + generator covers 24 of 3,505 unmigrated keys and says "0.7%", listing + the largest groups it did not touch. A plan that covered a fraction + while implying completeness would be worse than no plan. + + Operations are emitted as `upsert` with `matchOn: ["name"]`, so a plan + can be re-run — an operator will run it more than once — and the + supplement is applied *after* `export.json` rather than merged into it, + so a generated mapping can never override one the official script got + right. - Stage new systemd unit / Compose file changes without activating them. ### 4.4 Recovery-mode migration @@ -577,13 +599,16 @@ happens to need them. `preflight.DeploymentKind` is a type alias for script is Stalwart's, not ours — need a policy for what happens when it changes upstream (re-vendor + re-test before bumping the pin, never silently float to `main`). -- **Settings apply-plan (§4.3): now the critical path, not an enhancement.** - Measured against production, `migrate_v016.py` carries 1.8% of the - settings; `server.listener` is not among them, so a migrated instance - answers on no ports until the rest is rebuilt. Building this from - `unmigrated.txt` is what would make both a meaningful rehearsal and a - working cutover possible. It should still require explicit operator - sign-off even with `--yes` set for everything else. +- **Settings apply-plan (§4.3): started, and the critical path.** + `internal/applyplan` covers `server.listener` — verified end to end by + applying a generated plan to a real 0.16.14 instance and reading back all + ten listeners with correct protocols, binds and TLS flags. Everything + else in the worklist is still manual: the largest groups are + `lookup.url-redirectors`, `lookup.trusted-domains`, `spam-filter.list`, + `spam-filter.rule` and `spam-filter.dnsbl`, each needing its own + confirmed mapping (`x:StoreLookup`, `x:HttpLookup`, `x:SpamRule`, + `x:SpamDnsblServer`). The generated plan should still require explicit + operator sign-off even with `--yes` set for everything else. - **Account/mailbox enumeration** (`stalwartapi.Client.AccountSnapshot`): **implemented**, including per-mailbox message counts. Account count and domains come from `x:Account/query` + `x:Account/get` against Stalwart's diff --git a/README.md b/README.md index b7cf1dc..3751c7e 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,18 @@ It copies no data, clones nothing, starts no server, and never writes to the store, so it is safe to run against production repeatedly and without a maintenance window. +It also generates a **supplemental plan** for the part it can rebuild +automatically — currently your network listeners, which is the difference +between a migrated server that answers and one that doesn't — and reports +exactly how much of the worklist that covers (on a test instance: 24 of +3,505 keys, and it says so rather than implying more). Review it, then apply +it after `export.json`: + +```sh +stalwart-cli apply --file /runs//supplement.json \ + --url https://mail.example.com +``` + Expect the worklist to be long. Measured against a real production instance, `migrate_v016.py` carried **219 of 12,401 settings — 1.8%**. The rest, including `server.listener`, has to be rebuilt by hand; until it is, a diff --git a/cmd/stalwart-migrate/rehearse.go b/cmd/stalwart-migrate/rehearse.go index 1c4e4db..fc209dc 100644 --- a/cmd/stalwart-migrate/rehearse.go +++ b/cmd/stalwart-migrate/rehearse.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" + "github.com/LINUXexpert-org/stalwart-migrator/internal/applyplan" "github.com/LINUXexpert-org/stalwart-migrator/internal/backup" "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" "github.com/LINUXexpert-org/stalwart-migrator/internal/plan" @@ -84,6 +85,7 @@ func runRehearse(args []string) (err error) { // files that aren't there. keptWorklist := filepath.Join(runStateDir, "unmigrated.txt") keptPlan := filepath.Join(runStateDir, "export.json") + keptSupplement := filepath.Join(runStateDir, "supplement.json") defer func() { if _, statErr := os.Stat(runWorkDir); os.IsNotExist(statErr) { return @@ -211,8 +213,17 @@ func runRehearse(args []string) (err error) { } } fmt.Printf("\n !! %s\n", unmigrated.Summary(10)) - fmt.Println(" These do not carry over. Rebuild them on the migrated instance before it serves mail -") - fmt.Println(" note that server.listener is typically among them, so until you do, it answers on nothing.") + fmt.Println(" These do not carry over on their own. See the supplemental plan below for the part") + fmt.Println(" this tool can rebuild for you.") + } + + // Generate what we can of the gap (ARCHITECTURE.md §4.3). This is + // best-effort by design and reports its own coverage: the plan is + // worth having precisely because it is honest about how much of the + // worklist it does not touch. + fmt.Println("\n--- supplemental plan (best-effort) ---") + if err := generateSupplement(store, rs, settingsPath, unmigratedPath, keptSupplement); err != nil { + fmt.Fprintf(os.Stderr, "warning: couldn't generate the supplemental plan: %v\n", err) } if err := store.Save(rs); err != nil { return fmt.Errorf("save run state: %w", err) @@ -244,3 +255,54 @@ func copyFile(src, dst string) error { } return out.Sync() } + +// generateSupplement builds the best-effort apply plan for settings +// migrate_v016.py left behind, writes it beside the run's other +// conclusions, and reports honestly how much of the gap it closed. +// +// It is deliberately applied *after* export.json rather than merged into +// it: the official conversion is the authority on everything it handles, +// and a generated plan that overlapped it could silently override a +// correct mapping with a guessed one. +func generateSupplement(store *checkpoint.Store, rs *checkpoint.RunState, settingsPath, unmigratedPath, outPath string) error { + settings, err := backup.ReadSettingsDump(settingsPath) + if err != nil { + return err + } + unmigrated, err := backup.ReadUnmigratedKeys(unmigratedPath, settings) + if err != nil { + return err + } + + plan, coverage, err := applyplan.Build(settings, unmigrated, applyplan.DefaultGenerators()) + if err != nil { + return err + } + if len(plan.Operations) == 0 { + fmt.Println("nothing this tool can rebuild automatically yet - the whole worklist is manual") + return nil + } + if err := plan.WriteNDJSON(outPath); err != nil { + return err + } + if sum, size, hashErr := backup.HashFile(outPath); hashErr == nil { + rs.RecordArtifact("supplemental-plan", checkpoint.Artifact{Path: outPath, SHA256: sum, SizeBytes: size}) + } + + fmt.Printf("%s\n", coverage.Summary(6)) + for _, w := range coverage.Warnings { + fmt.Printf(" warning: %s\n", w) + } + fmt.Printf("\n supplement: %s\n", outPath) + fmt.Println(" Review it, then apply it after export.json:") + fmt.Printf(" stalwart-cli apply --file %s --url \n", outPath) + fmt.Println(" It is generated, not authoritative - read it before you run it.") + + _, err = store.RunStep(rs, checkpoint.PhaseStage, "supplemental-plan", func() (checkpoint.StepOutcome, error) { + return checkpoint.StepOutcome{ + Detail: fmt.Sprintf("generated %d operation(s) covering %d of %d unmigrated setting(s)", + len(plan.Operations), coverage.CoveredKeys, coverage.TotalKeys), + }, nil + }) + return err +} diff --git a/internal/applyplan/doc.go b/internal/applyplan/doc.go new file mode 100644 index 0000000..a451348 --- /dev/null +++ b/internal/applyplan/doc.go @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package applyplan generates a best-effort v0.16 apply plan for the settings migrate_v016.py does not carry over. +// See ARCHITECTURE.md §4.3 for the design. +package applyplan diff --git a/internal/applyplan/listener.go b/internal/applyplan/listener.go new file mode 100644 index 0000000..df03abc --- /dev/null +++ b/internal/applyplan/listener.go @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package applyplan + +import ( + "fmt" + "sort" + "strings" +) + +// ListenerGenerator maps v0.15's server.listener.* settings onto v0.16 +// x:NetworkListener objects. +// +// This is the first generator built, and deliberately so: server.listener +// is not among the settings migrate_v016.py carries over, so a migrated +// instance binds nothing and answers on no port until these exist. Every +// other unmigrated setting degrades the server; this one stops it being a +// server at all. +// +// The v0.16 shape below was not taken from the published schema reference, +// which gives `bind` as a JSON array. That form is rejected by the actual +// 0.16.14 binary ("Invalid value for object property. Properties: bind"). +// The encoding here - a value-keyed set, {"[::]:25": true} - was confirmed +// by applying a plan to a real recovery-mode 0.16.14 instance and reading +// the result back with `stalwart-cli snapshot`: +// +// {"@type":"upsert","object":"NetworkListener","matchOn":["name"], +// "value":{"...":{"name":"t2","bind":{"[::]:2526":true}, +// "protocol":"smtp","tlsImplicit":false, ...}}} +type ListenerGenerator struct{} + +func (ListenerGenerator) Prefix() string { return "server.listener." } + +// protocolMap translates v0.15 protocol values to v0.16's enum. Only +// manageSieve actually differs - it is camelCase in v0.16 and lowercase in +// v0.15 - but going through an explicit table means an unrecognized value +// is reported rather than passed through to be rejected at apply time. +var protocolMap = map[string]string{ + "smtp": "smtp", + "lmtp": "lmtp", + "http": "http", + "imap": "imap", + "pop3": "pop3", + "managesieve": "manageSieve", +} + +func (g ListenerGenerator) Generate(settings map[string]string) ([]Operation, []string, []string, error) { + type listener struct { + binds []string + protocol string + tlsImplicit *bool + keys []string + } + byName := map[string]*listener{} + + get := func(name string) *listener { + if byName[name] == nil { + byName[name] = &listener{} + } + return byName[name] + } + + var warnings []string + for key, value := range settings { + rest := strings.TrimPrefix(key, g.Prefix()) + name, field, ok := strings.Cut(rest, ".") + if !ok || name == "" { + continue + } + l := get(name) + switch { + case field == "bind" || strings.HasPrefix(field, "bind."): + // v0.15 allows either a single bind or numbered bind.0000 + // entries; v0.16 holds them all in one set. + if value != "" { + l.binds = append(l.binds, value) + l.keys = append(l.keys, key) + } + case field == "protocol": + l.protocol = strings.ToLower(strings.TrimSpace(value)) + l.keys = append(l.keys, key) + case field == "tls.implicit": + implicit := strings.EqualFold(strings.TrimSpace(value), "true") + l.tlsImplicit = &implicit + l.keys = append(l.keys, key) + default: + // Deliberately not covered: anything else under a listener + // (socket tuning, per-listener TLS overrides) is left to the + // operator rather than guessed at, and stays counted as + // unhandled so the coverage report tells the truth. + } + } + + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + sort.Strings(names) + + var ops []Operation + var covered []string + for _, name := range names { + l := byName[name] + if len(l.binds) == 0 { + warnings = append(warnings, fmt.Sprintf("listener %q has no bind address in the source settings - skipped", name)) + continue + } + protocol, known := protocolMap[l.protocol] + if l.protocol == "" { + // v0.16's own default; recorded as a warning because inferring + // a listener's protocol is not something to do silently. + protocol = "smtp" + warnings = append(warnings, fmt.Sprintf("listener %q declared no protocol - defaulting to smtp, which may be wrong", name)) + } else if !known { + warnings = append(warnings, fmt.Sprintf("listener %q uses protocol %q, which has no known v0.16 equivalent - skipped", name, l.protocol)) + continue + } + + bind := map[string]any{} + sort.Strings(l.binds) + for _, addr := range l.binds { + bind[addr] = true + } + value := map[string]any{ + "name": name, + "bind": bind, + "protocol": protocol, + } + if l.tlsImplicit != nil { + value["tlsImplicit"] = *l.tlsImplicit + } + + ops = append(ops, Operation{ + Type: "upsert", + Object: "NetworkListener", + MatchOn: []string{"name"}, + Value: map[string]map[string]any{"listener-" + name: value}, + }) + covered = append(covered, l.keys...) + } + sort.Strings(covered) + return ops, covered, warnings, nil +} + +// DefaultGenerators is the set Build runs. It is short on purpose: each +// entry is a mapping confirmed against a real v0.16 binary, and an +// unverified guess in here would produce a plan that fails at apply time or, +// worse, silently configures the wrong thing. +func DefaultGenerators() []Generator { + return []Generator{ListenerGenerator{}} +} diff --git a/internal/applyplan/listener_test.go b/internal/applyplan/listener_test.go new file mode 100644 index 0000000..48c09a7 --- /dev/null +++ b/internal/applyplan/listener_test.go @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package applyplan + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// productionListeners is the exact server.listener.* key set a real +// Stalwart 0.15.5 reports, taken verbatim from a live instance. +var productionListeners = map[string]string{ + "server.listener.http.bind": "[::]:8090", + "server.listener.http.protocol": "http", + "server.listener.https.bind": "[::]:443", + "server.listener.https.protocol": "http", + "server.listener.https.tls.implicit": "true", + "server.listener.imap.bind": "[::]:143", + "server.listener.imap.protocol": "imap", + "server.listener.imaptls.bind": "[::]:993", + "server.listener.imaptls.protocol": "imap", + "server.listener.imaptls.tls.implicit": "true", + "server.listener.sieve.bind": "[::]:4190", + "server.listener.sieve.protocol": "managesieve", + "server.listener.smtp.bind": "[::]:25", + "server.listener.smtp.protocol": "smtp", + "server.listener.submissions.bind": "[::]:465", + "server.listener.submissions.protocol": "smtp", + "server.listener.submissions.tls.implicit": "true", +} + +func generate(t *testing.T, settings map[string]string) ([]Operation, []string, []string) { + t.Helper() + ops, covered, warnings, err := ListenerGenerator{}.Generate(settings) + if err != nil { + t.Fatalf("Generate: %v", err) + } + return ops, covered, warnings +} + +func findListener(ops []Operation, name string) map[string]any { + for _, op := range ops { + for _, v := range op.Value { + if v["name"] == name { + return v + } + } + } + return nil +} + +func TestListenerGeneratorMapsAProductionListenerSet(t *testing.T) { + ops, covered, warnings := generate(t, productionListeners) + + if len(ops) != 7 { + t.Errorf("generated %d listener(s), want 7", len(ops)) + } + if len(warnings) != 0 { + t.Errorf("unexpected warnings for a well-formed listener set: %v", warnings) + } + if len(covered) != len(productionListeners) { + t.Errorf("covered %d keys, want all %d - uncovered keys must not be silently dropped", + len(covered), len(productionListeners)) + } + + smtp := findListener(ops, "smtp") + if smtp == nil { + t.Fatal("no smtp listener generated") + } + // The encoding the real 0.16.14 binary accepts: a value-keyed set, not + // the array the published docs show. + bind, ok := smtp["bind"].(map[string]any) + if !ok { + t.Fatalf("bind is %T, want a value-keyed set - an array is rejected by the server", smtp["bind"]) + } + if v, present := bind["[::]:25"]; !present || v != true { + t.Errorf("bind = %v, want {\"[::]:25\": true}", bind) + } +} + +// managesieve -> manageSieve is the one protocol whose spelling changes, +// and getting it wrong fails at apply time rather than at generation. +func TestListenerGeneratorRenamesManageSieve(t *testing.T) { + ops, _, _ := generate(t, productionListeners) + sieve := findListener(ops, "sieve") + if sieve == nil { + t.Fatal("no sieve listener generated") + } + if sieve["protocol"] != "manageSieve" { + t.Errorf("protocol = %v, want manageSieve (v0.16 spelling)", sieve["protocol"]) + } +} + +func TestListenerGeneratorCarriesImplicitTLS(t *testing.T) { + ops, _, _ := generate(t, productionListeners) + for name, want := range map[string]bool{"imaptls": true, "submissions": true, "https": true} { + l := findListener(ops, name) + if l == nil { + t.Fatalf("no %s listener generated", name) + } + if l["tlsImplicit"] != want { + t.Errorf("%s tlsImplicit = %v, want %v", name, l["tlsImplicit"], want) + } + } + // A listener that never mentioned TLS shouldn't have an opinion forced + // onto it; v0.16's own default applies. + if smtp := findListener(ops, "smtp"); smtp != nil { + if _, present := smtp["tlsImplicit"]; present { + t.Error("smtp listener should not assert tlsImplicit when the source didn't set it") + } + } +} + +// Upsert keyed on name, so re-running a plan updates rather than colliding. +// An operator will run this more than once. +func TestListenerGeneratorEmitsIdempotentUpserts(t *testing.T) { + ops, _, _ := generate(t, productionListeners) + for _, op := range ops { + if op.Type != "upsert" { + t.Errorf("@type = %q, want upsert so the plan can be re-run", op.Type) + } + if len(op.MatchOn) != 1 || op.MatchOn[0] != "name" { + t.Errorf("matchOn = %v, want [name]", op.MatchOn) + } + } +} + +func TestListenerGeneratorRefusesUnknownProtocol(t *testing.T) { + ops, covered, warnings := generate(t, map[string]string{ + "server.listener.weird.bind": "[::]:9999", + "server.listener.weird.protocol": "gopher", + }) + if len(ops) != 0 { + t.Errorf("generated %d op(s) for an unmappable protocol, want 0 - guessing here fails at apply time", len(ops)) + } + if len(covered) != 0 { + t.Errorf("covered = %v, want none: an unmapped listener must stay counted as unhandled", covered) + } + if len(warnings) != 1 || !strings.Contains(warnings[0], "gopher") { + t.Errorf("warnings = %v, want one naming the unknown protocol", warnings) + } +} + +func TestListenerGeneratorWarnsOnMissingBindAndProtocol(t *testing.T) { + _, _, warnings := generate(t, map[string]string{"server.listener.orphan.protocol": "smtp"}) + if len(warnings) != 1 || !strings.Contains(warnings[0], "no bind address") { + t.Errorf("warnings = %v, want one about the missing bind address", warnings) + } + + ops, _, warnings := generate(t, map[string]string{"server.listener.mystery.bind": "[::]:26"}) + if len(ops) != 1 { + t.Fatalf("generated %d op(s), want 1 with a defaulted protocol", len(ops)) + } + if len(warnings) != 1 || !strings.Contains(warnings[0], "may be wrong") { + t.Errorf("warnings = %v, want one flagging the defaulted protocol", warnings) + } +} + +func TestBuildReportsHonestCoverage(t *testing.T) { + settings := map[string]string{} + unmigrated := map[string]bool{} + for k, v := range productionListeners { + settings[k] = v + unmigrated[k] = true + } + // Things no generator handles yet, in the proportions a real instance + // shows: the plan must not imply it covered them. + for i := 0; i < 500; i++ { + k := "spam-filter.rule.r" + string(rune('a'+i%26)) + string(rune('a'+i/26)) + settings[k] = "x" + unmigrated[k] = true + } + + plan, coverage, err := Build(settings, unmigrated, DefaultGenerators()) + if err != nil { + t.Fatal(err) + } + if coverage.CoveredKeys != len(productionListeners) { + t.Errorf("CoveredKeys = %d, want %d", coverage.CoveredKeys, len(productionListeners)) + } + if coverage.TotalKeys != len(unmigrated) { + t.Errorf("TotalKeys = %d, want %d", coverage.TotalKeys, len(unmigrated)) + } + if len(plan.Operations) == 0 { + t.Fatal("no operations generated") + } + summary := coverage.Summary(5) + if !strings.Contains(summary, "still unhandled") || !strings.Contains(summary, "spam-filter.rule") { + t.Errorf("summary must name what it did NOT cover:\n%s", summary) + } + if strings.Contains(summary, "100.0%") { + t.Errorf("summary claims full coverage but most settings are unhandled:\n%s", summary) + } +} + +// Build must never generate for a key migrate_v016.py already handles, or +// the plan would fight the official conversion. +func TestBuildIgnoresSettingsTheOfficialScriptAlreadyMigrates(t *testing.T) { + settings := map[string]string{} + for k, v := range productionListeners { + settings[k] = v + } + plan, coverage, err := Build(settings, map[string]bool{}, DefaultGenerators()) + if err != nil { + t.Fatal(err) + } + if len(plan.Operations) != 0 { + t.Errorf("generated %d op(s) for settings not listed as unmigrated, want 0", len(plan.Operations)) + } + if coverage.CoveredKeys != 0 { + t.Errorf("CoveredKeys = %d, want 0", coverage.CoveredKeys) + } +} + +func TestWriteNDJSONIsOnePlanEntryPerLine(t *testing.T) { + ops, _, _ := generate(t, productionListeners) + path := filepath.Join(t.TempDir(), "plan.json") + plan := &Plan{Operations: ops} + if err := plan.WriteNDJSON(path); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != len(ops) { + t.Fatalf("wrote %d line(s) for %d operation(s)", len(lines), len(ops)) + } + for i, line := range lines { + var op Operation + if err := json.Unmarshal([]byte(line), &op); err != nil { + t.Errorf("line %d is not valid JSON: %v", i+1, err) + } + if op.Object != "NetworkListener" { + t.Errorf("line %d object = %q", i+1, op.Object) + } + } +} diff --git a/internal/applyplan/plan.go b/internal/applyplan/plan.go new file mode 100644 index 0000000..1757d65 --- /dev/null +++ b/internal/applyplan/plan.go @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package applyplan + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" +) + +// Operation is one entry in a stalwart-cli apply plan. The plan file is +// NDJSON - one operation per line - which is the shape migrate_v016.py's +// own export.json uses and what `stalwart-cli apply --file` consumes. +// +// Generated operations are "upsert" with MatchOn rather than "create", +// so re-running a plan against an instance that already has some of these +// objects updates them instead of failing on a duplicate. An operator will +// run this more than once - after a failed cutover, or while iterating on +// the parts the generator can't cover - and a plan that only works on a +// pristine instance would be a trap. +type Operation struct { + Type string `json:"@type"` + Object string `json:"object"` + MatchOn []string `json:"matchOn,omitempty"` + Value map[string]map[string]any `json:"value"` +} + +// Plan is the generated apply plan plus an account of what it covers. +type Plan struct { + Operations []Operation + // Covered are the v0.15 setting keys the operations above account for. + Covered []string + // Warnings are per-setting notes where a mapping was possible but + // lossy or assumed - surfaced rather than silently applied. + Warnings []string +} + +// WriteNDJSON writes the plan in the format `stalwart-cli apply --file` +// reads. +func (p *Plan) WriteNDJSON(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) + if err != nil { + return fmt.Errorf("applyplan: create %s: %w", path, err) + } + defer f.Close() + enc := json.NewEncoder(f) + for _, op := range p.Operations { + if err := enc.Encode(op); err != nil { + return fmt.Errorf("applyplan: write %s: %w", path, err) + } + } + return f.Sync() +} + +// Generator turns one family of v0.15 settings into v0.16 objects. +// +// Each generator declares the key prefix it consumes and is handed every +// setting under it. Returning the keys it consumed is what lets Build +// report honest coverage: a generator that quietly skips half its input +// would otherwise look like full coverage of its prefix. +type Generator interface { + // Prefix is the v0.15 key prefix this generator claims, e.g. + // "server.listener.". + Prefix() string + // Generate maps the settings under Prefix into operations, returning + // the keys it actually accounted for. + Generate(settings map[string]string) (ops []Operation, covered []string, warnings []string, err error) +} + +// Coverage is the honest accounting of a generated plan: what it handles, +// and what an operator still has to rebuild by hand. +// +// This matters more than the plan itself. ARCHITECTURE.md §4.3 is explicit +// that this generation is best-effort, and against a real instance the +// unmigrated set runs to five figures. A plan that covered a tenth of it +// while implying completeness would be worse than no plan at all. +type Coverage struct { + TotalKeys int + CoveredKeys int + Remaining []PrefixCount + Warnings []string + ObjectsByType map[string]int +} + +// PrefixCount is one group of still-unhandled settings. +type PrefixCount struct { + Prefix string + Keys int +} + +// Summary renders the coverage for an operator, largest gaps first. +func (c *Coverage) Summary(maxPrefixes int) string { + var b strings.Builder + pct := 0.0 + if c.TotalKeys > 0 { + pct = 100 * float64(c.CoveredKeys) / float64(c.TotalKeys) + } + fmt.Fprintf(&b, "generated a plan for %d of %d unmigrated setting(s) (%.1f%%)", c.CoveredKeys, c.TotalKeys, pct) + if len(c.ObjectsByType) > 0 { + types := make([]string, 0, len(c.ObjectsByType)) + for t := range c.ObjectsByType { + types = append(types, t) + } + sort.Strings(types) + parts := make([]string, 0, len(types)) + for _, t := range types { + parts = append(parts, fmt.Sprintf("%d %s", c.ObjectsByType[t], t)) + } + fmt.Fprintf(&b, ": %s", strings.Join(parts, ", ")) + } + if len(c.Remaining) > 0 { + fmt.Fprintf(&b, "\n still unhandled, largest first:") + shown := c.Remaining + if len(shown) > maxPrefixes { + shown = shown[:maxPrefixes] + } + for _, r := range shown { + fmt.Fprintf(&b, "\n %-32s %d keys", r.Prefix, r.Keys) + } + if len(c.Remaining) > len(shown) { + fmt.Fprintf(&b, "\n ... and %d more prefix(es)", len(c.Remaining)-len(shown)) + } + } + return b.String() +} + +// Build runs every registered generator over the unmigrated settings and +// returns the plan together with its coverage. +// +// settings is the full v0.15 settings dump; unmigrated is the set of keys +// migrate_v016.py reported it did not carry over. Only keys in unmigrated +// are considered: anything the official script already handles must not be +// generated a second time, or the plan would fight the conversion. +func Build(settings map[string]string, unmigrated map[string]bool, generators []Generator) (*Plan, *Coverage, error) { + plan := &Plan{} + coverage := &Coverage{TotalKeys: len(unmigrated), ObjectsByType: map[string]int{}} + covered := map[string]bool{} + + for _, g := range generators { + subset := map[string]string{} + for k, v := range settings { + if unmigrated[k] && strings.HasPrefix(k, g.Prefix()) { + subset[k] = v + } + } + if len(subset) == 0 { + continue + } + ops, keys, warnings, err := g.Generate(subset) + if err != nil { + return nil, nil, fmt.Errorf("applyplan: %s: %w", g.Prefix(), err) + } + plan.Operations = append(plan.Operations, ops...) + plan.Warnings = append(plan.Warnings, warnings...) + coverage.Warnings = append(coverage.Warnings, warnings...) + for _, op := range ops { + coverage.ObjectsByType[op.Object] += len(op.Value) + } + for _, k := range keys { + covered[k] = true + } + } + + for k := range covered { + plan.Covered = append(plan.Covered, k) + } + sort.Strings(plan.Covered) + coverage.CoveredKeys = len(covered) + + remaining := map[string]int{} + for k := range unmigrated { + if covered[k] { + continue + } + remaining[groupPrefix(k)]++ + } + for prefix, n := range remaining { + coverage.Remaining = append(coverage.Remaining, PrefixCount{Prefix: prefix, Keys: n}) + } + sort.Slice(coverage.Remaining, func(i, j int) bool { + if coverage.Remaining[i].Keys != coverage.Remaining[j].Keys { + return coverage.Remaining[i].Keys > coverage.Remaining[j].Keys + } + return coverage.Remaining[i].Prefix < coverage.Remaining[j].Prefix + }) + return plan, coverage, nil +} + +// groupPrefix reduces a setting key to the first two dotted segments, which +// is how migrate_v016.py's own unmigrated.txt groups them - keeping the two +// reports comparable side by side. +func groupPrefix(key string) string { + parts := strings.Split(key, ".") + if len(parts) <= 2 { + return key + } + return parts[0] + "." + parts[1] +} diff --git a/internal/backup/settingsdump.go b/internal/backup/settingsdump.go index a7aef8e..079a7e3 100644 --- a/internal/backup/settingsdump.go +++ b/internal/backup/settingsdump.go @@ -7,6 +7,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -263,3 +264,51 @@ func ReadUnmigratedReport(path string) (*UnmigratedReport, error) { sort.Slice(report.Prefixes, func(i, j int) bool { return report.Prefixes[i].Keys > report.Prefixes[j].Keys }) return report, nil } + +// ReadSettingsDump loads the flat {key: value} settings map +// migrate_v016.py's dump step writes. +func ReadSettingsDump(path string) (map[string]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("backup: read settings dump %s: %w", path, err) + } + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("backup: parse settings dump %s: %w", path, err) + } + settings := make(map[string]string, len(raw)) + for k, v := range raw { + if s, ok := v.(string); ok { + settings[k] = s + continue + } + settings[k] = fmt.Sprint(v) + } + return settings, nil +} + +// ReadUnmigratedKeys returns the set of settings keys migrate_v016.py +// reported it did not carry over. +// +// unmigrated.txt lists prefixes and counts rather than individual keys, so +// this expands those prefixes against the settings dump. That is why it +// needs both files: the report says "spam-filter.rule: 424 keys", and only +// the dump knows which 424. +func ReadUnmigratedKeys(reportPath string, settings map[string]string) (map[string]bool, error) { + report, err := ReadUnmigratedReport(reportPath) + if err != nil { + return nil, err + } + keys := map[string]bool{} + if report == nil { + return keys, nil + } + for _, p := range report.Prefixes { + for k := range settings { + if k == p.Prefix || strings.HasPrefix(k, p.Prefix+".") { + keys[k] = true + } + } + } + return keys, nil +}