From 9faa21f4f1e2433e2867fd355bbfb19330f0e279 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 23 Aug 2026 23:20:47 -0700 Subject: [PATCH] Fix the three defects that cost a production restore A live migration on 2026-08-24 stopped a production mail server and then discovered the host's stalwart-cli was 0.13.4 - present, but from when the CLI shipped with the server, with no `apply` command. The migration needs v1.0.2+ from the separately-versioned stalwartlabs/cli repository. Recovery was closed in both directions. v0.16's recovery-mode boot had already bumped the store schema to v6, so the 0.15.5 binary refused to reopen it ("expected 5 or below, found 6"). Going forward needed export.json, which this tool's own failure path had deleted - and regenerating it required a settings dump from a live v0.15 instance that could no longer start. The operator restored a day-old snapshot and lost a day of mail across nine domains. Three fixes: 1. preflight.CheckExternalTools verifies stalwart-cli exists and is v1.0.2 or later, and that python3 runs - before anything is touched. Every fact needed to prevent this was available in under a second from a stopped state. Skipped for a patch upgrade, which invokes neither tool. 2. A failed run no longer deletes its work directory. Cleaning up on every exit path was right for a sandboxed rehearsal and catastrophic here: once the service is stopped the settings dump cannot be regenerated, so deleting it removes the only way forward. The failure now prints the resume command instead. 3. `run --resume ` continues an interrupted run. The checkpoint machinery existed but never engaged, because run created a new run every invocation - so a retry re-ran preflight against a binary already moved aside, and failed. Completed steps are skipped from the checkpoint. Proven against a VM built to match the failure: stalwart-cli 0.15.5, accounts and mail seeded. * preflight refused, service still active, mail still accepted * a stub CLI passing --version and failing apply left the run stopped with all eight inputs intact and the resume command printed * --resume carried it to a clean finish: five seconds of downtime, listeners regenerated, admin role restored, quotas rebuilt That failure-path test is the one that should have run before production. Every earlier test had stalwart-cli installed from the start, and the one failure I did exercise happened to leave its artifacts behind. --- ARCHITECTURE.md | 34 +++++++ cmd/stalwart-migrate/preflight.go | 4 + cmd/stalwart-migrate/rehearse.go | 7 ++ cmd/stalwart-migrate/run.go | 36 +++++++- internal/cutover/unit.go | 14 ++- internal/cutover/unit_test.go | 37 ++++++++ internal/preflight/checks.go | 27 ++++-- internal/preflight/checks_test.go | 8 ++ internal/preflight/dependencies.go | 112 ++++++++++++++++++++++++ internal/preflight/dependencies_test.go | 102 +++++++++++++++++++++ 10 files changed, 370 insertions(+), 11 deletions(-) create mode 100644 internal/preflight/dependencies.go create mode 100644 internal/preflight/dependencies_test.go diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 898f068..1875d2e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -829,6 +829,40 @@ happens to need them. `preflight.DeploymentKind` is a type alias for - `tenant-admin` has no v0.16 equivalent and is reported as unrestorable rather than silently dropped. +- **A live migration attempt failed and cost a restore. Three defects, all + fixed, all now proven against a reproduction.** On 2026-08-24 a real + migration stopped a production mail server and then discovered the host's + `stalwart-cli` was 0.13.4 - present, but from when the CLI shipped with + the server, and with no `apply` command. Recovery was closed in both + directions: v0.16's recovery-mode boot had already bumped the store schema + to v6 (`expected 5 or below, found 6`), so the old binary could not reopen + it, and going forward needed `export.json`, which the failure path had + deleted - regenerating it required a settings dump from a live v0.15 + instance that could no longer start. The operator restored a day-old + snapshot and lost a day of mail. + + - **Preflight now verifies the external tools** (`CheckExternalTools`), + before anything is touched: stalwart-cli must exist and be v1.0.2 or + later, and python3 must run. Every fact needed to prevent that outage + was available in under a second from a stopped state. Skipped entirely + for a patch upgrade, which invokes neither tool. + - **A failed run no longer deletes its own inputs.** The cleanup applied + to every exit path, which was right for a sandboxed rehearsal and + catastrophic here: after the service is stopped, the settings dump + cannot be regenerated, so deleting it removes the only way forward. + - **`run --resume ` continues an interrupted run.** The checkpoint + machinery existed but never engaged, because `run` created a new run + every invocation - so a retry would re-run preflight against a binary + already moved aside and fail. Completed steps are now skipped. + + Verified on a VM built to match: stalwart-cli 0.15.5 installed, accounts + and mail seeded. Preflight refused with the service still running and mail + still flowing; a stub CLI that passed the version check and failed the + apply left the run stopped with all eight inputs intact; and `--resume` + carried it to a clean finish - five seconds of downtime, quotas rebuilt. + That failure-path test is the one that should have run before production, + and did not. + - **`run` is built and works.** preflight -> stage -> dump -> preserve -> stop -> convert -> supplement -> recovery-mode -> cutover, checkpointed throughout, verified end to end against a real 0.15.5: mail down for six diff --git a/cmd/stalwart-migrate/preflight.go b/cmd/stalwart-migrate/preflight.go index ff6b564..591427b 100644 --- a/cmd/stalwart-migrate/preflight.go +++ b/cmd/stalwart-migrate/preflight.go @@ -26,6 +26,8 @@ func runPreflight(args []string) error { targetVersion := fs.String("target", "latest", `target Stalwart version, or "latest"`) minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size") stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in") + pythonPath := fs.String("python", "python3", "path to python3, needed by migrate_v016.py") + stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", "path to stalwart-cli (v1.0.2 or later; a separate download from the server)") if err := fs.Parse(args); err != nil { return err } @@ -46,6 +48,8 @@ func runPreflight(args []string) error { AdminPassword: *adminPassword, TargetVersion: *targetVersion, MinFreeMultiple: *minFree, + CLIPath: *stalwartCLI, + PythonPath: *pythonPath, }) report, err := checker.Run(context.Background(), store, rs) diff --git a/cmd/stalwart-migrate/rehearse.go b/cmd/stalwart-migrate/rehearse.go index a9bc139..dfa7f59 100644 --- a/cmd/stalwart-migrate/rehearse.go +++ b/cmd/stalwart-migrate/rehearse.go @@ -52,6 +52,8 @@ func runRehearse(args []string) (err error) { stateDir := fs.String("state-dir", checkpoint.DefaultBaseDir, "directory to store run checkpoints in") workDir := fs.String("work-dir", "/var/lib/stalwart-migrator/work", "scratch directory for the dumps and converted plan (cleaned up afterward - see --keep-artifacts)") pythonPath := fs.String("python", "python3", "path to python3") + stalwartCLI := fs.String("stalwart-cli", "stalwart-cli", + "path to stalwart-cli (v1.0.2 or later; a separate download from the server) - rehearsal doesn't invoke it, but checks it so `run` doesn't fail after stopping the service") migrationScriptSHA256 := fs.String("migration-script-sha256", "", "pinned sha256 of migrate_v016.py (recommended; the first run prints the hash to pin)") minFree := fs.Float64("min-free-multiple", 2.0, "free-space multiple preflight checks for; rehearsal itself copies nothing") keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/ afterward") @@ -94,6 +96,10 @@ func runRehearse(args []string) (err error) { fmt.Printf("\nartifacts kept at %s (--keep-artifacts)\n", runWorkDir) return } + if err != nil { + fmt.Fprintf(os.Stderr, "\nthe rehearsal failed; its artifacts are kept at %s for inspection\n", runWorkDir) + return + } if rmErr := os.RemoveAll(runWorkDir); rmErr != nil { fmt.Fprintf(os.Stderr, "\nwarning: failed to clean up %s: %v (remove it manually)\n", runWorkDir, rmErr) return @@ -106,6 +112,7 @@ func runRehearse(args []string) (err error) { BinaryPath: *binaryPath, ConfigPath: *configPath, DataDir: *dataDir, ContainerName: *containerName, AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword, TargetVersion: *targetVersion, MinFreeMultiple: *minFree, HTTPClient: httpClient, + CLIPath: *stalwartCLI, PythonPath: *pythonPath, }) pfReport, err := checker.Run(ctx, store, rs) fmt.Print(pfReport.String()) diff --git a/cmd/stalwart-migrate/run.go b/cmd/stalwart-migrate/run.go index 4e1b7e8..b05119a 100644 --- a/cmd/stalwart-migrate/run.go +++ b/cmd/stalwart-migrate/run.go @@ -69,6 +69,7 @@ func runRun(args []string) (err error) { minFree := fs.Float64("min-free-multiple", 2.0, "required free disk space as a multiple of the data directory size") recalcQuotas := fs.Bool("recalculate-quotas", true, "schedule the post-migration quota rebuild") keepArtifacts := fs.Bool("keep-artifacts", false, "don't delete work-dir/ afterward") + resume := fs.String("resume", "", "resume an interrupted run by id instead of starting a new one (see `status` for ids)") yes := fs.Bool("yes", false, "actually perform the migration") recoveryConfirmed := fs.Bool("recovery-point-confirmed", false, "confirm you have a snapshot or backup you have verified you can restore - this tool cannot undo a migration") @@ -104,11 +105,25 @@ func runRun(args []string) (err error) { } store := checkpoint.NewStore(*stateDir) - rs, err := store.Create("", *targetVersion) - if err != nil { - return fmt.Errorf("create run: %w", err) + var rs *checkpoint.RunState + if *resume != "" { + // Resuming is not a convenience. A run that fails partway leaves + // the service stopped and the store part-migrated, and starting + // over is often impossible: preflight would re-run against a + // binary that has already been moved aside, and the settings dump + // needs a live pre-migration instance that no longer exists. + // Completed steps are skipped from the checkpoint, so this picks + // up where it stopped. + if rs, err = store.Load(*resume); err != nil { + return fmt.Errorf("resume run %s: %w", *resume, err) + } + fmt.Printf("\nresuming run: %s (completed steps will be skipped)\n", rs.RunID) + } else { + if rs, err = store.Create("", *targetVersion); err != nil { + return fmt.Errorf("create run: %w", err) + } + fmt.Printf("\nrun id: %s\n", rs.RunID) } - fmt.Printf("\nrun id: %s\n", rs.RunID) runWorkDir := filepath.Join(*workDir, rs.RunID) runStateDir := filepath.Join(*stateDir, rs.RunID) if err := os.MkdirAll(runWorkDir, 0o750); err != nil { @@ -119,6 +134,18 @@ func runRun(args []string) (err error) { fmt.Printf("\nartifacts kept at %s (--keep-artifacts)\n", runWorkDir) return } + if err != nil { + // Never clean up after a failure. These files - the settings + // dump, the converted config and export plan - are the run's + // inputs, and after the service has been stopped they cannot + // be regenerated: the dump needs a live pre-migration instance. + // Deleting them once turned a missing-dependency error into a + // restore-from-snapshot, because there was no way forward and + // no way back. + fmt.Fprintf(os.Stderr, "\nthe run failed; its artifacts are kept at %s\n", runWorkDir) + fmt.Fprintf(os.Stderr, "resume it once the cause is fixed:\n stalwart-migrate run --resume %s [same flags]\n", rs.RunID) + return + } if rmErr := os.RemoveAll(runWorkDir); rmErr != nil { fmt.Fprintf(os.Stderr, "warning: couldn't clean up %s: %v\n", runWorkDir, rmErr) } @@ -129,6 +156,7 @@ func runRun(args []string) (err error) { BinaryPath: *binaryPath, ConfigPath: *configPath, DataDir: *dataDir, ContainerName: *containerName, AdminURL: *adminURL, AdminUser: *adminUser, AdminPassword: *adminPassword, TargetVersion: *targetVersion, MinFreeMultiple: *minFree, HTTPClient: httpClient, + CLIPath: *stalwartCLI, PythonPath: *pythonPath, }).Run(ctx, store, rs) fmt.Print(pfReport.String()) if err != nil { diff --git a/internal/cutover/unit.go b/internal/cutover/unit.go index 26117f1..e153a8a 100644 --- a/internal/cutover/unit.go +++ b/internal/cutover/unit.go @@ -97,10 +97,20 @@ func rewriteExecStart(line, binaryPath, configPath string) (string, error) { if configPath != "" { replaced := false - for i := 0; i < len(fields)-1; i++ { - if fields[i] == "--config" || fields[i] == "-c" { + for i := 0; i < len(fields); i++ { + switch { + // Both spellings are real, and getting this wrong is not + // cosmetic: a production unit uses the equals form, and only + // matching the separated one appended a second --config, so + // the service started with two and used the wrong one - the + // v0.15 config, which v0.16 cannot read as a store descriptor. + case strings.HasPrefix(fields[i], "--config=") || strings.HasPrefix(fields[i], "-c="): + fields[i] = "--config=" + configPath + replaced = true + case (fields[i] == "--config" || fields[i] == "-c") && i+1 < len(fields): fields[i+1] = configPath replaced = true + i++ } } if !replaced { diff --git a/internal/cutover/unit_test.go b/internal/cutover/unit_test.go index 3ffba8e..6ee9de1 100644 --- a/internal/cutover/unit_test.go +++ b/internal/cutover/unit_test.go @@ -128,3 +128,40 @@ func TestRewriteUnitHandlesMultipleExecStartLines(t *testing.T) { t.Fatalf("an empty ExecStart= names no executable and should be refused, got:\n%s", got) } } + +// A real production unit writes ExecStart=... --config=/path. Only matching +// the separated "--config /path" form appended a second flag, leaving the +// service started with two configs and using the v0.15 one - which v0.16 +// cannot read as a store descriptor. Found while preparing a live +// migration, before it ran. +func TestRewriteUnitReplacesTheEqualsFormConfig(t *testing.T) { + unit := "[Service]\nExecStart=/opt/stalwart/bin/stalwart --config=/opt/stalwart/etc/config.toml\n" + got, err := RewriteUnit(unit, "/opt/stalwart/bin/stalwart", "/opt/stalwart/etc/config.json") + if err != nil { + t.Fatal(err) + } + if strings.Count(got, "--config") != 1 { + t.Errorf("expected exactly one --config argument, got:\n%s", got) + } + if !strings.Contains(got, "--config=/opt/stalwart/etc/config.json") { + t.Errorf("equals-form config not replaced:\n%s", got) + } + if strings.Contains(got, "config.toml") { + t.Errorf("the old config path survived:\n%s", got) + } +} + +// The separated form must keep working; both spellings are real. +func TestRewriteUnitReplacesTheSeparatedFormConfig(t *testing.T) { + unit := "[Service]\nExecStart=/usr/local/bin/stalwart --config /etc/stalwart/config.toml\n" + got, err := RewriteUnit(unit, "/usr/local/bin/stalwart", "/etc/stalwart/config.json") + if err != nil { + t.Fatal(err) + } + if strings.Count(got, "--config") != 1 { + t.Errorf("expected exactly one --config argument, got:\n%s", got) + } + if !strings.Contains(got, "--config /etc/stalwart/config.json") { + t.Errorf("separated-form config not replaced:\n%s", got) + } +} diff --git a/internal/preflight/checks.go b/internal/preflight/checks.go index 367c023..62b3a72 100644 --- a/internal/preflight/checks.go +++ b/internal/preflight/checks.go @@ -27,7 +27,12 @@ type Options struct { AdminPassword string TargetVersion string // e.g. "0.16.14" or "latest" MinFreeMultiple float64 - HTTPClient *http.Client + // CLIPath and PythonPath are the external programs the migration + // shells out to. Checked before anything is touched - see + // CheckExternalTools. + CLIPath string + PythonPath string + HTTPClient *http.Client } // Checker runs the preflight checks described in ARCHITECTURE.md §4.1. @@ -107,7 +112,7 @@ func (c *Checker) Run(ctx context.Context, store *checkpoint.Store, rs *checkpoi return report, err } - if _, err := runCheck("upgrade-direction", func() (CheckResult, string) { + boundaryOutcome, err := runCheck("upgrade-direction", func() (CheckResult, string) { curV, errCur := parseSemver(versionOutcome.Extra) tgtV, errTgt := parseSemver(targetOutcome.Extra) if errCur != nil || errTgt != nil { @@ -123,15 +128,27 @@ func (c *Checker) Run(ctx context.Context, store *checkpoint.Store, rs *checkpoi return CheckResult{ Status: StatusOK, Detail: fmt.Sprintf("%s -> %s crosses the 0.15/0.16 major boundary: full recovery-mode migration plan required (ARCHITECTURE.md §4.4)", curV, tgtV), - }, "" + }, "crosses" } return CheckResult{ Status: StatusOK, Detail: fmt.Sprintf("%s -> %s is a same-boundary patch upgrade: fast-path plan applies (ARCHITECTURE.md §4.6)", curV, tgtV), - }, "" - }); err != nil { + }, "patch" + }) + if err != nil { return report, err } + crossesBoundary := boundaryOutcome.Extra != "patch" + + // Before anything else that matters: are the tools this migration + // depends on actually here? Discovering a missing stalwart-cli after + // the service has been stopped is what this exists to prevent. + for _, res := range CheckExternalTools(ctx, c.opts.CLIPath, c.opts.PythonPath, crossesBoundary) { + result := res + if _, err := runCheck(result.Name, func() (CheckResult, string) { return result, "" }); err != nil { + return report, err + } + } deploymentOutcome, err := runCheck("deployment-kind", func() (CheckResult, string) { kind := DetectDeploymentKind(ctx, c.opts.ContainerName) diff --git a/internal/preflight/checks_test.go b/internal/preflight/checks_test.go index 8bc5aa7..d2e97d0 100644 --- a/internal/preflight/checks_test.go +++ b/internal/preflight/checks_test.go @@ -44,6 +44,10 @@ func countLines(t *testing.T, path string) int { } func TestCheckerRunEndToEndAndResume(t *testing.T) { + // preflight now verifies the external tools before anything else; give + // this environment ones that satisfy it. + fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12") + fakeTool(t, "python3", "Python 3.13.5") // -- fixtures -------------------------------------------------------- counterPath := filepath.Join(t.TempDir(), "invocations") binaryPath := writeFakeBinary(t, "0.15.5", counterPath) @@ -200,6 +204,10 @@ func TestCheckerRunFlagsInsufficientDiskSpace(t *testing.T) { } func TestCheckerRunCapturesAccountSnapshotWhenAdminURLSet(t *testing.T) { + // preflight now verifies the external tools before anything else; give + // this environment ones that satisfy it. + fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12") + fakeTool(t, "python3", "Python 3.13.5") counterPath := filepath.Join(t.TempDir(), "invocations") binaryPath := writeFakeBinary(t, "0.15.5", counterPath) diff --git a/internal/preflight/dependencies.go b/internal/preflight/dependencies.go new file mode 100644 index 0000000..c09b3a8 --- /dev/null +++ b/internal/preflight/dependencies.go @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package preflight + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" +) + +// minCLIVersion is the oldest stalwart-cli the migration can use. +// UPGRADING/v0_16.md is explicit: "install the new CLI (make sure to +// install v1.0.2 or later)". Older builds have no `apply` command at all. +var minCLIVersion = semver{1, 0, 2} + +// CheckExternalTools verifies the programs the migration shells out to +// exist and are new enough, before anything is touched. +// +// This check exists because its absence took a production mail server down. +// A live migration stopped the service, converted the settings, and only +// then discovered that the host's stalwart-cli was 0.13.4 - present, but +// from the era when the CLI shipped with the server, and with no `apply` +// command. The migration needs v1.0.2+ from the separately-versioned +// stalwartlabs/cli repository, which is a different download most operators +// have never made. Recovery cost a restore from a day-old snapshot. +// +// Every one of those facts was knowable in under a second, from a stopped +// state, before any risk was taken. That is what preflight is for. +func CheckExternalTools(ctx context.Context, cliPath, pythonPath string, crossesMajorBoundary bool) []CheckResult { + if cliPath == "" { + cliPath = "stalwart-cli" + } + if pythonPath == "" { + pythonPath = "python3" + } + if !crossesMajorBoundary { + // A patch bump replays no settings, so neither tool is invoked. + return []CheckResult{{ + Name: "external-tools", + Status: StatusOK, + Detail: "not needed for a same-boundary patch upgrade: no settings conversion or apply happens", + }} + } + + var results []CheckResult + + version, err := toolVersion(ctx, cliPath, "--version") + switch { + case err != nil: + results = append(results, CheckResult{ + Name: "stalwart-cli", + Status: StatusFail, + Detail: fmt.Sprintf("%s is required to replay settings into the migrated store and could not be run: %v. "+ + "It is a separate download from the server - see https://stalw.art/docs/management/cli/overview - and "+ + "must be v%s or later. A stalwart-cli that shipped alongside a 0.15 server has no `apply` command and "+ + "will not do", cliPath, err, minCLIVersion), + }) + default: + parsed, parseErr := parseSemver(version) + switch { + case parseErr != nil: + results = append(results, CheckResult{ + Name: "stalwart-cli", + Status: StatusWarn, + Detail: fmt.Sprintf("%s reported a version this tool couldn't parse (%q); it must be v%s or later", cliPath, version, minCLIVersion), + }) + case parsed.Compare(minCLIVersion) < 0: + results = append(results, CheckResult{ + Name: "stalwart-cli", + Status: StatusFail, + Detail: fmt.Sprintf("%s is v%s, but the migration needs v%s or later - older builds have no `apply` command. "+ + "Install it from https://stalw.art/docs/management/cli/overview (it is a separate download from the server)", + cliPath, parsed, minCLIVersion), + }) + default: + results = append(results, CheckResult{ + Name: "stalwart-cli", + Status: StatusOK, + Detail: fmt.Sprintf("%s is v%s (needs v%s or later)", cliPath, parsed, minCLIVersion), + }) + } + } + + if _, err := toolVersion(ctx, pythonPath, "--version"); err != nil { + results = append(results, CheckResult{ + Name: "python3", + Status: StatusFail, + Detail: fmt.Sprintf("%s is required to run Stalwart's migrate_v016.py and could not be run: %v", pythonPath, err), + }) + } else { + results = append(results, CheckResult{ + Name: "python3", Status: StatusOK, + Detail: fmt.Sprintf("%s is available for migrate_v016.py", pythonPath), + }) + } + return results +} + +// toolVersion runs a program's version flag and returns its output. +func toolVersion(ctx context.Context, path string, arg string) (string, error) { + cmd := exec.CommandContext(ctx, path, arg) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("%w (output: %s)", err, strings.TrimSpace(out.String())) + } + return strings.TrimSpace(out.String()), nil +} diff --git a/internal/preflight/dependencies_test.go b/internal/preflight/dependencies_test.go new file mode 100644 index 0000000..2a0e4b5 --- /dev/null +++ b/internal/preflight/dependencies_test.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package preflight + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// fakeTool puts an executable of the given name at the front of PATH, +// reporting the given --version output. +func fakeTool(t *testing.T, name, versionOutput string) { + t.Helper() + dir := t.TempDir() + script := "#!/bin/sh\necho '" + versionOutput + "'\n" + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func statusOf(results []CheckResult, name string) (Status, string) { + for _, r := range results { + if r.Name == name { + return r.Status, r.Detail + } + } + return "", "" +} + +// This is the check whose absence took a production mail server down: a +// live migration stopped the service, then discovered the host's +// stalwart-cli was 0.13.4 and had no `apply` command. Recovery cost a +// restore from a day-old snapshot. +func TestCheckExternalToolsFailsWhenTheCLIIsMissing(t *testing.T) { + t.Setenv("PATH", t.TempDir()) // nothing on PATH at all + results := CheckExternalTools(context.Background(), "", "", true) + + status, detail := statusOf(results, "stalwart-cli") + if status != StatusFail { + t.Errorf("stalwart-cli = %q, want fail - this must stop the run before the service does", status) + } + if !strings.Contains(detail, "separate download") { + t.Errorf("detail %q should say it's a separate download from the server", detail) + } + if !strings.Contains(detail, "v1.0.2") { + t.Errorf("detail %q should name the minimum version", detail) + } +} + +// The exact version that was on the production host. It exists, it runs, +// and it cannot do the job. +func TestCheckExternalToolsFailsOnTheOldBundledCLI(t *testing.T) { + fakeTool(t, "stalwart-cli", "stalwart-cli 0.13.4") + fakeTool(t, "python3", "Python 3.13.5") + results := CheckExternalTools(context.Background(), "", "", true) + + status, detail := statusOf(results, "stalwart-cli") + if status != StatusFail { + t.Errorf("stalwart-cli 0.13.4 = %q, want fail", status) + } + if !strings.Contains(detail, "0.13.4") || !strings.Contains(detail, "no `apply` command") { + t.Errorf("detail %q should name the version found and why it won't do", detail) + } +} + +func TestCheckExternalToolsAcceptsASupportedCLI(t *testing.T) { + fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12") + fakeTool(t, "python3", "Python 3.13.5") + results := CheckExternalTools(context.Background(), "", "", true) + + for _, name := range []string{"stalwart-cli", "python3"} { + if status, detail := statusOf(results, name); status != StatusOK { + t.Errorf("%s = %q (%s), want ok", name, status, detail) + } + } +} + +func TestCheckExternalToolsFailsWithoutPython(t *testing.T) { + fakeTool(t, "stalwart-cli", "stalwart-cli 1.0.12") + results := CheckExternalTools(context.Background(), "", "/nonexistent/python3", true) + if status, _ := statusOf(results, "python3"); status != StatusFail { + t.Errorf("python3 = %q, want fail - migrate_v016.py cannot run without it", status) + } +} + +// A patch bump replays no settings, so neither tool is invoked and a +// missing CLI must not block it. +func TestCheckExternalToolsSkipsForAPatchUpgrade(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + results := CheckExternalTools(context.Background(), "", "", false) + if len(results) != 1 { + t.Fatalf("got %d result(s), want a single skip-style result", len(results)) + } + if results[0].Status != StatusOK { + t.Errorf("status = %q, want ok for a patch upgrade with no tools present", results[0].Status) + } +}