diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 346bd2a..ed1149f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -340,9 +340,26 @@ no-data-loss guarantee that was not measured. See `internal/validate/content_integrity.go`. -Runs automatically after cutover; failure here stops the run, reports -loudly, and exits non-zero, leaving the operator to decide what to restore -(§4.8). +Runs automatically after cutover, against the service cutover has just +started: that is the instance people will actually use — its real config, +its real ports, under its real service manager — and checking it costs no +extra downtime, where booting a second copy inside the maintenance window +would. Failure stops the run, reports loudly, and exits non-zero, leaving +the operator to decide what to restore (§4.8). The service is deliberately +left running: by this point the store has been migrated in place, so +stopping it undoes nothing. + +A check that could not be performed is reported as **skipped**, never as a +pass. Preflight only captures the "before" snapshot when it has an admin URL +to capture it from, and a run without one has to say it compared nothing +rather than imply everything survived. + +`internal/validate.BootCheck` remains the equivalent for an instance the +tool boots itself, which is what `rehearse` needs; `run` uses `RunLive`. + +Of the checks listed below, what exists today is the account/domain +comparison. The rest are the intended shape of the suite, not a description +of it. - **Version check**: reported server version matches the target exactly. - **Auth check**: WebUI login succeeds over the *configured hostname* via diff --git a/README.md b/README.md index 119ff22..3a7e52f 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,11 @@ has been performed end to end. | `stalwart-migrate rehearse` | **Works** — read-only; converts your settings and reports what won't carry over | | `stalwart-migrate run` | **Works** — performs the migration; `--recovery-point-confirmed --yes` | | `stalwart-migrate status ` | **Works** | -| `stalwart-migrate report ` | Not implemented | +| `stalwart-migrate report ` | **Works** — prints what validation found for a run | **`run` performs the migration**, in the order -preflight → stage → dump → stop → convert → recovery-mode → cutover. It +preflight → stage → dump → stop → convert → recovery-mode → cutover → +validate. It needs two flags: `--yes` (intent) and `--recovery-point-confirmed` (a claim that you have a snapshot or backup you have verified you can restore — this tool cannot undo a migration and will not start without it). @@ -37,6 +38,14 @@ Measured on a full migration: the store converts in seconds, and the service was down for **6 seconds** end to end. Plan the window around verification, not data volume. +After cutover, `run` compares the migrated instance against the snapshot +preflight took, and fails the command if an account or a domain that existed +before is missing from it. The service is left running either way — by that +point the store has been migrated in place, so stopping it would not undo +anything; your recovery point is the way back. `report ` prints the +same finding again later. Where preflight had no admin URL to snapshot from, +validation reports itself as skipped rather than passed. + Package state: Lines are implementation only; each package carries its tests alongside. diff --git a/cmd/stalwart-migrate/main.go b/cmd/stalwart-migrate/main.go index 4636737..11d9e1b 100644 --- a/cmd/stalwart-migrate/main.go +++ b/cmd/stalwart-migrate/main.go @@ -30,7 +30,7 @@ func main() { case "status": err = runStatus(os.Args[2:]) case "report": - err = fmt.Errorf("not implemented yet: see internal/validate") + err = runReport(os.Args[2:]) default: usage() os.Exit(1) @@ -50,5 +50,5 @@ commands: rehearse convert this instance's settings and report what will NOT carry over (read-only) run perform the migration (needs --yes and --recovery-point-confirmed) status show the state of an in-progress or completed run - report print the validation report for a run (not implemented yet)`) + report print the validation report for a run`) } diff --git a/cmd/stalwart-migrate/report.go b/cmd/stalwart-migrate/report.go new file mode 100644 index 0000000..48d405b --- /dev/null +++ b/cmd/stalwart-migrate/report.go @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "flag" + "fmt" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" + "github.com/LINUXexpert-org/stalwart-migrator/internal/validate" +) + +// runReport prints what validation found for a run, from the checkpoint the +// run already wrote. It re-reads rather than re-checks: the comparison is +// against a pre-migration snapshot, so running it again later would answer a +// different question — how the instance looks now, not how it looked when it +// was migrated. +func runReport(args []string) error { + fs := flag.NewFlagSet("report", flag.ExitOnError) + stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory runs are checkpointed in") + + runID, rest := splitRunID(fs, args) + if err := fs.Parse(rest); err != nil { + return err + } + if fs.NArg() != 0 || runID == "" { + return fmt.Errorf("usage: stalwart-migrate report [flags]") + } + + store := checkpoint.NewStore(*stateDir) + rs, err := store.Load(runID) + if err != nil { + return fmt.Errorf("load run %s: %w", runID, err) + } + + report := reportFromSteps(rs) + fmt.Printf("run: %s\n", rs.RunID) + fmt.Printf("source: %s\n", rs.SourceVersion) + fmt.Printf("target: %s\n", rs.TargetVersion) + + if len(report.Results) == 0 { + fmt.Println("\nno validation has been recorded for this run.") + if rs.PreflightSnapshot == nil { + fmt.Println("preflight captured no pre-migration snapshot, so there was nothing to compare against;") + fmt.Println("pass --admin-url to preflight next time and the comparison becomes possible.") + } else { + fmt.Println("the run did not reach the validate phase - see `stalwart-migrate status " + runID + "`.") + } + return nil + } + + fmt.Println("\nvalidation:") + fmt.Print(report.String()) + if report.Blocking() { + // Non-zero so this is usable in a script that gates on it, and so a + // failed migration cannot look successful to anything watching. + return fmt.Errorf("validation recorded a failure for run %s", runID) + } + return nil +} + +// reportFromSteps rebuilds the validation report from the checkpointed +// steps, so the report survives the process that produced it. +func reportFromSteps(rs *checkpoint.RunState) validate.Report { + var report validate.Report + for _, step := range rs.Steps { + if step.Phase != checkpoint.PhaseValidate { + continue + } + status := validate.StatusOK + switch { + case step.Error != "": + status = validate.StatusFail + case step.Verdict == string(validate.StatusFail): + status = validate.StatusFail + case step.Verdict == string(validate.StatusSkip): + status = validate.StatusSkip + case step.Status != checkpoint.StepDone: + // Recorded but never completed: the run stopped partway. + status = validate.StatusFail + } + detail := step.Detail + if step.Error != "" { + if detail != "" { + detail += " " + } + detail += "(error: " + step.Error + ")" + } + report.Results = append(report.Results, validate.CheckResult{Name: step.Name, Status: status, Detail: detail}) + } + return report +} diff --git a/cmd/stalwart-migrate/report_test.go b/cmd/stalwart-migrate/report_test.go new file mode 100644 index 0000000..586c9ed --- /dev/null +++ b/cmd/stalwart-migrate/report_test.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "testing" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" + "github.com/LINUXexpert-org/stalwart-migrator/internal/validate" +) + +// `report` re-reads what the run recorded rather than re-checking, so the +// mapping from checkpointed step back to verdict is the whole command. The +// case that matters is a failure staying a failure: a migration that lost an +// account must not read as clean the next morning. + +func TestReportFromSteps(t *testing.T) { + for _, tc := range []struct { + name string + step checkpoint.StepRecord + want validate.Status + block bool + }{ + { + name: "a completed comparison is a pass", + step: checkpoint.StepRecord{Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepDone, StepOutcome: checkpoint.StepOutcome{Detail: "12 accounts checked"}}, + want: validate.StatusOK, + }, + { + name: "a failing verdict survives the round trip", + step: checkpoint.StepRecord{Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepDone, StepOutcome: checkpoint.StepOutcome{Verdict: string(validate.StatusFail), Detail: "missing accounts: bob@example.org"}}, + want: validate.StatusFail, + block: true, + }, + { + name: "a skipped check stays skipped, not a pass", + step: checkpoint.StepRecord{Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepDone, StepOutcome: checkpoint.StepOutcome{Verdict: string(validate.StatusSkip), Detail: "no snapshot"}}, + want: validate.StatusSkip, + }, + { + name: "a step that errored is a failure", + step: checkpoint.StepRecord{Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepFailed, Error: "connection refused"}, + want: validate.StatusFail, + block: true, + }, + { + name: "a step that never finished is a failure, not a pass", + step: checkpoint.StepRecord{Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepRunning}, + want: validate.StatusFail, + block: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := reportFromSteps(&checkpoint.RunState{Steps: []checkpoint.StepRecord{tc.step}}) + if len(got.Results) != 1 { + t.Fatalf("got %d results, want 1", len(got.Results)) + } + if got.Results[0].Status != tc.want { + t.Fatalf("status = %q, want %q", got.Results[0].Status, tc.want) + } + if got.Blocking() != tc.block { + t.Fatalf("Blocking() = %v, want %v", got.Blocking(), tc.block) + } + }) + } +} + +func TestReportFromStepsIgnoresOtherPhases(t *testing.T) { + rs := &checkpoint.RunState{Steps: []checkpoint.StepRecord{ + {Phase: checkpoint.PhaseCutover, Name: "start-service", Status: checkpoint.StepDone}, + {Phase: checkpoint.PhasePreflight, Name: "disk-space", Status: checkpoint.StepFailed, Error: "nope"}, + {Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepDone, StepOutcome: checkpoint.StepOutcome{Detail: "ok"}}, + }} + got := reportFromSteps(rs) + if len(got.Results) != 1 || got.Results[0].Name != "content-integrity" { + t.Fatalf("expected only the validate phase, got %+v", got.Results) + } + // A failed preflight step must not leak into the validation verdict. + if got.Blocking() { + t.Fatal("a failure in another phase must not make validation look failed") + } +} + +func TestReportFromStepsOnARunThatNeverValidated(t *testing.T) { + rs := &checkpoint.RunState{Steps: []checkpoint.StepRecord{ + {Phase: checkpoint.PhaseCutover, Name: "start-service", Status: checkpoint.StepDone}, + }} + if got := reportFromSteps(rs); len(got.Results) != 0 { + t.Fatalf("expected no results, got %+v", got.Results) + } +} + +func TestReportFromStepsKeepsTheError(t *testing.T) { + rs := &checkpoint.RunState{Steps: []checkpoint.StepRecord{ + {Phase: checkpoint.PhaseValidate, Name: "content-integrity", Status: checkpoint.StepFailed, StepOutcome: checkpoint.StepOutcome{Detail: "reached the instance"}, Error: "401 unauthorized"}, + }} + got := reportFromSteps(rs) + if d := got.Results[0].Detail; d != "reached the instance (error: 401 unauthorized)" { + t.Fatalf("detail = %q, want it to carry both the detail and the error", d) + } +} diff --git a/cmd/stalwart-migrate/run.go b/cmd/stalwart-migrate/run.go index d603705..a5f310d 100644 --- a/cmd/stalwart-migrate/run.go +++ b/cmd/stalwart-migrate/run.go @@ -21,6 +21,7 @@ import ( "github.com/LINUXexpert-org/stalwart-migrator/internal/recovery" "github.com/LINUXexpert-org/stalwart-migrator/internal/service" "github.com/LINUXexpert-org/stalwart-migrator/internal/stage" + "github.com/LINUXexpert-org/stalwart-migrator/internal/validate" ) // runRun implements `stalwart-migrate run`: the real migration. @@ -334,6 +335,24 @@ func runRun(args []string) (err error) { return fmt.Errorf("cutover failed: %w", err) } + fmt.Println("\n--- validate ---") + valReport, valErr := validate.RunLive(ctx, store, rs, validate.LiveOptions{ + AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword, + HTTPClient: httpClient, Before: rs.PreflightSnapshot, + }) + fmt.Print(valReport.String()) + if valErr != nil { + return fmt.Errorf("the migrated service is up, but validation could not complete - check it by hand before "+ + "treating this migration as done: %w", valErr) + } + if valReport.Blocking() { + // The service is live and serving mail; that is deliberately not + // undone here. The operator has the finding and their recovery + // point, and only they can weigh one against the other. + return fmt.Errorf("the migrated service is up, but accounts or domains from before the migration are missing " + + "from it - see the FAIL line above. Your recovery point is the way back; this tool will not undo a migration") + } + fmt.Printf("\nMIGRATION COMPLETE for run %s. Mail was down for %s.\n", rs.RunID, time.Since(windowStart).Round(time.Second)) fmt.Printf("Now: confirm you can log in as %s, send and receive a test message, and work through\n", *adminUser) diff --git a/internal/validate/live.go b/internal/validate/live.go new file mode 100644 index 0000000..c347ca9 --- /dev/null +++ b/internal/validate/live.go @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package validate + +import ( + "context" + "fmt" + "net/http" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" + "github.com/LINUXexpert-org/stalwart-migrator/internal/stalwartapi" +) + +// LiveOptions describes the migrated instance cutover has just started. +type LiveOptions struct { + AdminURL string + AdminUser string + AdminPassword string + HTTPClient *http.Client + + // Before is what preflight captured before anything was touched + // (checkpoint.RunState.PreflightSnapshot). Nil when preflight had no + // admin URL to capture it from, in which case there is nothing to + // compare against and the check reports that rather than passing. + Before *checkpoint.PreflightSnapshot +} + +// CheckLive compares a running instance against the pre-migration snapshot. +// +// The same comparison BootCheck performs against an instance it booted +// itself, aimed instead at the service cutover has already started. That is +// the instance people will actually use — its real config, its real ports, +// under its real service manager — and checking it costs no extra downtime, +// where booting a second copy inside the maintenance window would. +func CheckLive(ctx context.Context, client *stalwartapi.Client, before *checkpoint.PreflightSnapshot) (*ContentIntegrityResult, error) { + return compareContentIntegrity(ctx, client, before) +} + +// RunLive executes the post-cutover content comparison as a checkpointed +// step, mirroring how every other phase records itself. +// +// A missing snapshot or admin URL is reported as skipped, never as a pass: +// "every account survived" and "we were unable to look" are different +// answers, and ARCHITECTURE.md §4.7 is explicit that this suite must not +// imply a guarantee it did not measure. +func RunLive(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts LiveOptions) (Report, error) { + var report Report + + switch { + case opts.AdminURL == "": + report.Results = append(report.Results, CheckResult{ + Name: "content-integrity", Status: StatusSkip, + Detail: "no admin URL configured - nothing could be compared against the migrated instance", + }) + return report, nil + case opts.Before == nil: + report.Results = append(report.Results, CheckResult{ + Name: "content-integrity", Status: StatusSkip, + Detail: "preflight captured no pre-migration snapshot - there is nothing to compare the migrated instance against", + }) + return report, nil + } + + client := &stalwartapi.Client{ + BaseURL: opts.AdminURL, Username: opts.AdminUser, Password: opts.AdminPassword, HTTPClient: opts.HTTPClient, + } + + outcome, err := store.RunStep(rs, checkpoint.PhaseValidate, "content-integrity", func() (checkpoint.StepOutcome, error) { + r, err := CheckLive(ctx, client, opts.Before) + if err != nil { + return checkpoint.StepOutcome{}, err + } + if !r.OK() { + // Recorded as a completed step with a failing verdict rather + // than an error: the comparison ran, and its answer is the + // finding. An error here would read as "we could not look". + return checkpoint.StepOutcome{Verdict: string(StatusFail), Detail: r.String()}, nil + } + return checkpoint.StepOutcome{Detail: r.String()}, nil + }) + if err != nil { + report.Results = append(report.Results, CheckResult{ + Name: "content-integrity", Status: StatusFail, + Detail: fmt.Sprintf("could not compare the migrated instance against the pre-migration snapshot: %v", err), + }) + return report, err + } + + status := StatusOK + if outcome.Verdict == string(StatusFail) { + status = StatusFail + } + report.Results = append(report.Results, CheckResult{Name: "content-integrity", Status: status, Detail: outcome.Detail}) + return report, nil +} diff --git a/internal/validate/live_test.go b/internal/validate/live_test.go new file mode 100644 index 0000000..b632e6e --- /dev/null +++ b/internal/validate/live_test.go @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package validate + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" +) + +// A run that never captured a "before" cannot be checked against one. The +// point of these two is that such a run reports as unchecked rather than as +// passing: ARCHITECTURE.md §4.7 is explicit that this suite must not imply a +// guarantee it did not measure. + +func TestRunLiveSkipsWithoutSnapshot(t *testing.T) { + store, rs := newRun(t) + report, err := RunLive(context.Background(), store, rs, LiveOptions{AdminURL: "https://mail.example.org", Before: nil}) + if err != nil { + t.Fatalf("RunLive: %v", err) + } + if got := report.Results[0].Status; got != StatusSkip { + t.Fatalf("status = %q, want %q", got, StatusSkip) + } + if report.Blocking() { + t.Fatal("a skipped check must not block the run") + } + if !strings.Contains(report.Results[0].Detail, "nothing to compare") { + t.Fatalf("detail should say why it was skipped, got %q", report.Results[0].Detail) + } +} + +func TestRunLiveSkipsWithoutAdminURL(t *testing.T) { + store, rs := newRun(t) + report, err := RunLive(context.Background(), store, rs, LiveOptions{Before: &checkpoint.PreflightSnapshot{}}) + if err != nil { + t.Fatalf("RunLive: %v", err) + } + if got := report.Results[0].Status; got != StatusSkip { + t.Fatalf("status = %q, want %q", got, StatusSkip) + } +} + +func TestRunLivePassesWhenEverythingSurvived(t *testing.T) { + srv := fakeInstance(t, []string{"example.org"}, map[string]float64{"ann@example.org": 10, "bob@example.org": 20}) + defer srv.Close() + + store, rs := newRun(t) + report, err := RunLive(context.Background(), store, rs, LiveOptions{ + AdminURL: srv.URL, AdminUser: "admin", AdminPassword: "pw", HTTPClient: srv.Client(), + Before: &checkpoint.PreflightSnapshot{ + Domains: []string{"example.org"}, + UsedQuota: map[string]int64{"ann@example.org": 1, "bob@example.org": 2}, + }, + }) + if err != nil { + t.Fatalf("RunLive: %v", err) + } + if report.Blocking() { + t.Fatalf("expected a pass, got: %s", report.String()) + } + if got := report.Results[0].Status; got != StatusOK { + t.Fatalf("status = %q, want %q", got, StatusOK) + } +} + +func TestRunLiveFailsWhenAnAccountIsMissing(t *testing.T) { + // bob did not make it across. + srv := fakeInstance(t, []string{"example.org"}, map[string]float64{"ann@example.org": 10}) + defer srv.Close() + + store, rs := newRun(t) + report, err := RunLive(context.Background(), store, rs, LiveOptions{ + AdminURL: srv.URL, AdminUser: "admin", AdminPassword: "pw", HTTPClient: srv.Client(), + Before: &checkpoint.PreflightSnapshot{ + Domains: []string{"example.org"}, + UsedQuota: map[string]int64{"ann@example.org": 1, "bob@example.org": 2}, + }, + }) + // The comparison ran and found something: that is a finding, not an + // error, so the step completes and the report carries the verdict. + if err != nil { + t.Fatalf("RunLive returned an error for a completed comparison: %v", err) + } + if !report.Blocking() { + t.Fatalf("a missing account must block, got: %s", report.String()) + } + if !strings.Contains(report.Results[0].Detail, "bob@example.org") { + t.Fatalf("the report should name the missing account, got %q", report.Results[0].Detail) + } + // And it must be recorded, so `report ` can say so afterwards. + step := rs.Outcome(checkpoint.PhaseValidate, "content-integrity") + if step.Verdict != string(StatusFail) { + t.Fatalf("checkpoint verdict = %q, want %q", step.Verdict, StatusFail) + } +} + +func TestRunLiveFailsWhenADomainIsMissing(t *testing.T) { + srv := fakeInstance(t, []string{"example.org"}, map[string]float64{"ann@example.org": 10}) + defer srv.Close() + + store, rs := newRun(t) + report, _ := RunLive(context.Background(), store, rs, LiveOptions{ + AdminURL: srv.URL, AdminUser: "admin", AdminPassword: "pw", HTTPClient: srv.Client(), + Before: &checkpoint.PreflightSnapshot{ + Domains: []string{"example.org", "vanished.example"}, + UsedQuota: map[string]int64{"ann@example.org": 1}, + }, + }) + if !report.Blocking() { + t.Fatalf("a missing domain must block, got: %s", report.String()) + } + if !strings.Contains(report.Results[0].Detail, "vanished.example") { + t.Fatalf("the report should name the missing domain, got %q", report.Results[0].Detail) + } +} + +func TestRunLiveFailsWhenTheInstanceCannotBeRead(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + defer srv.Close() + + store, rs := newRun(t) + report, err := RunLive(context.Background(), store, rs, LiveOptions{ + AdminURL: srv.URL, AdminUser: "admin", AdminPassword: "pw", HTTPClient: srv.Client(), + Before: &checkpoint.PreflightSnapshot{Domains: []string{"example.org"}}, + }) + if err == nil { + t.Fatal("expected an error when the instance cannot be read") + } + if !report.Blocking() { + t.Fatalf("being unable to look must not read as a pass, got: %s", report.String()) + } +} + +func newRun(t *testing.T) (*checkpoint.Store, *checkpoint.RunState) { + t.Helper() + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatalf("create run: %v", err) + } + return store, rs +} + +// fakeInstance answers the principal listing the snapshot is built from. +func fakeInstance(t *testing.T, domains []string, accounts map[string]float64) *httptest.Server { + t.Helper() + items := make([]map[string]any, 0, len(domains)+len(accounts)) + for _, d := range domains { + items = append(items, map[string]any{"type": "domain", "name": d}) + } + for name, quota := range accounts { + items = append(items, map[string]any{"type": "individual", "name": name, "usedQuota": quota}) + } + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"items": items, "total": len(items)}}) + })) +} diff --git a/internal/validate/report.go b/internal/validate/report.go index a8764b8..7eda2e8 100644 --- a/internal/validate/report.go +++ b/internal/validate/report.go @@ -13,6 +13,11 @@ type Status string const ( StatusOK Status = "ok" StatusFail Status = "fail" + // StatusSkip is a check that could not be performed. It is deliberately + // not StatusOK: "every account survived" and "we were unable to look" + // are different answers, and reporting the second as the first is the + // failure mode ARCHITECTURE.md §4.7 warns about. + StatusSkip Status = "skip" ) type CheckResult struct {