Files
stalwart-migrator/internal/rollback/restore_test.go
T
jcoffey-dev ade9906275 Implement internal/rollback and the service control it needs
Rollback was the one phase gating everything else: `run` without --dry-run
refused because this tool could not undo a cutover it had committed to.
That reason is now gone, and the refusal has narrowed to the fact that
there is no real cutover to undo yet.

internal/rollback implements ARCHITECTURE.md 4.8 as eight checkpointed
steps under PhaseRollback: verify-backup, stop-service,
preserve-failed-state, restore-data, restore-binary,
restore-service-config, start-service, verify-rollback.

Three things depart from what 4.8 specified, each for a reason:

- The backup is re-verified against its manifest *before* the service is
  stopped, which the design didn't call out. Finding a corrupt backup is
  survivable while the failed instance is still up, and unsurvivable once
  its data directory has been moved aside.
- BuildPlan is separate from Run, so every reason to refuse (closed
  rollback window, FoundationDB, no recorded backup, unknown deployment
  kind, missing database credentials) is found before anything is touched.
  The CLI prints that resolved plan and acts only with --yes.
- The restore is re-verified against the same manifest after writing. A
  restore that put back truncated bytes and reported success would be
  worse than one that failed outright.

Nothing from the failed attempt is deleted: the half-migrated data
directory and the displaced binary are moved to .failed-<run-id> names, so
a retry after the underlying issue is fixed still has both the evidence and
the artifacts. Afterwards a reduced validation suite runs against the
*restored* instance (version, reachability, directory counts) rather than
assuming the restore worked.

internal/service is a new package holding the systemd/Docker control this
needs. It's separate rather than living inside internal/rollback because
cutover will need the identical operations, and because the commands that
can take mail delivery down belong in one auditable place - the same
reasoning that makes stalwartapi the only thing speaking JMAP.
preflight.DeploymentKind is now a type alias for service.Kind so detection
and control can't drift apart. Its Active() reads `systemctl is-active`'s
output rather than its exit status: systemctl exits non-zero for every
non-active state, so exit-status logic would make "inactive" - the answer a
rollback most needs - look like a failure to read the state at all.

Also fixes a pre-existing bug in `status`: Go's flag package stops parsing
at the first positional argument, so `status <run-id> --state-dir X` looked
the run up in the default directory and reported it missing. `rollback`
would have inherited the same footgun on a command whose flags decide what
gets overwritten.

Still open: `confirm` cannot set RollbackWindowClosed. Rollback honours the
flag and refuses when it's set, but closing the window is the point of no
return for the backups this restores from, so it should land with the
retention policy 6 describes rather than before it.

Verified end to end against a fake systemd deployment: half-migrated data
restored to its original contents, failed state preserved, old binary
reinstalled and reporting 0.15.5, unit restarted, and a re-run of the
completed rollback inert.
2026-08-23 15:54:48 -07:00

301 lines
10 KiB
Go

package rollback
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/LINUXexpert-org/stalwart-migrator/internal/backup"
)
// writeTree creates a small directory tree and returns its path, standing
// in for a Stalwart data directory.
func writeTree(t *testing.T, root string, files map[string]string) string {
t.Helper()
for rel, content := range files {
path := filepath.Join(root, rel)
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
t.Fatal(err)
}
}
return root
}
func TestPreserveFailedStateMovesRatherThanDeletes(t *testing.T) {
dir := t.TempDir()
dataDir := writeTree(t, filepath.Join(dir, "data"), map[string]string{"blobs/a": "half-migrated"})
preserved, moved, err := PreserveFailedState(dataDir, "run-1")
if err != nil {
t.Fatal(err)
}
if !moved {
t.Error("moved = false, want true")
}
if want := dataDir + ".failed-run-1"; preserved != want {
t.Errorf("preserved path = %q, want %q", preserved, want)
}
if _, err := os.Stat(dataDir); !os.IsNotExist(err) {
t.Errorf("original path still exists after being moved aside: %v", err)
}
if got := readFile(t, filepath.Join(preserved, "blobs/a")); got != "half-migrated" {
t.Errorf("preserved content = %q, want the failed attempt's data intact", got)
}
}
// A resumed rollback re-runs steps that were interrupted. If this clobbered
// the earlier rescue with whatever is at the original path the second time
// around, the failed attempt's state - the thing §4.8 promises never to
// delete - would be lost precisely when a rollback got interrupted.
func TestPreserveFailedStateDoesNotClobberAnEarlierRescue(t *testing.T) {
dir := t.TempDir()
dataDir := filepath.Join(dir, "data")
writeTree(t, dataDir, map[string]string{"a": "first"})
if _, _, err := PreserveFailedState(dataDir, "run-1"); err != nil {
t.Fatal(err)
}
writeTree(t, dataDir, map[string]string{"a": "second"})
preserved, moved, err := PreserveFailedState(dataDir, "run-1")
if err != nil {
t.Fatal(err)
}
if moved {
t.Error("moved = true on the second call, want false")
}
if got := readFile(t, filepath.Join(preserved, "a")); got != "first" {
t.Errorf("preserved content = %q, want %q - the first rescue must survive", got, "first")
}
}
func TestPreserveFailedStateIsFineWhenThereIsNothingToPreserve(t *testing.T) {
preserved, moved, err := PreserveFailedState(filepath.Join(t.TempDir(), "absent"), "run-1")
if err != nil {
t.Fatalf("want no error when the path doesn't exist, got %v", err)
}
if moved {
t.Error("moved = true, want false")
}
if preserved == "" {
t.Error("preserved path should still be reported")
}
}
func TestRestoreDataDirRestoresAndReverifies(t *testing.T) {
dir := t.TempDir()
src := writeTree(t, filepath.Join(dir, "data"), map[string]string{
"config": "settings", "blobs/one": "hello", "blobs/two": "world",
})
backupDir := filepath.Join(dir, "backup")
manifest, err := backup.CopyDataDir(src, backupDir)
if err != nil {
t.Fatal(err)
}
if err := os.RemoveAll(src); err != nil {
t.Fatal(err)
}
if err := RestoreDataDir(backupDir, src, manifest); err != nil {
t.Fatalf("RestoreDataDir: %v", err)
}
for rel, want := range map[string]string{"config": "settings", "blobs/one": "hello", "blobs/two": "world"} {
if got := readFile(t, filepath.Join(src, rel)); got != want {
t.Errorf("restored %s = %q, want %q", rel, got, want)
}
}
}
// A restore that put back corrupt bytes and reported success would be worse
// than one that failed: the operator would believe the rollback worked and
// only find out when Stalwart couldn't open its store.
func TestRestoreDataDirFailsWhenTheBackupNoLongerMatchesItsManifest(t *testing.T) {
dir := t.TempDir()
src := writeTree(t, filepath.Join(dir, "data"), map[string]string{"blobs/one": "hello"})
backupDir := filepath.Join(dir, "backup")
manifest, err := backup.CopyDataDir(src, backupDir)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupDir, "blobs/one"), []byte("corrupted"), 0o640); err != nil {
t.Fatal(err)
}
if err := os.RemoveAll(src); err != nil {
t.Fatal(err)
}
err = RestoreDataDir(backupDir, src, manifest)
if err == nil {
t.Fatal("RestoreDataDir: want error for a backup that no longer matches its manifest, got nil")
}
if !strings.Contains(err.Error(), "doesn't match the backup manifest") {
t.Errorf("error %q should say the restored directory didn't match the manifest", err)
}
}
func TestRestoreBinaryReinstallsOldAndPreservesCurrent(t *testing.T) {
dir := t.TempDir()
binaryPath := filepath.Join(dir, "stalwart")
preserved := binaryPath + ".v0.15.5"
if err := os.WriteFile(preserved, []byte("old binary"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(binaryPath, []byte("new binary"), 0o755); err != nil {
t.Fatal(err)
}
sum, _, err := hashFile(preserved)
if err != nil {
t.Fatal(err)
}
displaced, err := RestoreBinary(preserved, binaryPath, sum, "run-1")
if err != nil {
t.Fatalf("RestoreBinary: %v", err)
}
if got := readFile(t, binaryPath); got != "old binary" {
t.Errorf("binary at %s = %q, want the old one back", binaryPath, got)
}
if got := readFile(t, displaced); got != "new binary" {
t.Errorf("displaced binary = %q, want the new one preserved for a retry", got)
}
}
func TestRestoreBinaryRefusesAChangedBinary(t *testing.T) {
dir := t.TempDir()
binaryPath := filepath.Join(dir, "stalwart")
preserved := binaryPath + ".v0.15.5"
if err := os.WriteFile(preserved, []byte("swapped out from under us"), 0o755); err != nil {
t.Fatal(err)
}
_, err := RestoreBinary(preserved, binaryPath, "0000000000000000000000000000000000000000000000000000000000000000", "run-1")
if err == nil {
t.Fatal("RestoreBinary: want error when the preserved binary's checksum doesn't match, got nil")
}
if _, statErr := os.Stat(binaryPath); !os.IsNotExist(statErr) {
t.Error("a binary failing its checksum must not be installed")
}
}
func TestRestoreFileKeepsDestinationPermissions(t *testing.T) {
dir := t.TempDir()
src := filepath.Join(dir, "stalwart.service.preserved")
dst := filepath.Join(dir, "stalwart.service")
if err := os.WriteFile(src, []byte("[Service]\nExecStart=/usr/local/bin/stalwart\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(dst, []byte("[Service]\nExecStart=/usr/local/bin/stalwart-new\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := RestoreFile(src, dst); err != nil {
t.Fatalf("RestoreFile: %v", err)
}
if got := readFile(t, dst); !strings.Contains(got, "/usr/local/bin/stalwart\n") {
t.Errorf("restored unit = %q, want the preserved one", got)
}
info, err := os.Stat(dst)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o644 {
t.Errorf("restored unit mode = %v, want 0644 (the destination's own mode)", info.Mode().Perm())
}
// The temp file it writes through must not be left behind next to a
// service definition directory.
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(entries) != 2 {
t.Errorf("directory has %d entries, want 2 - a temp file was left behind", len(entries))
}
}
// psql's default is to report an error and keep going, which would let a
// half-applied restore exit zero and be reported as a successful rollback.
func TestBuildPsqlArgsStopsOnError(t *testing.T) {
args := BuildPsqlArgs(backup.SQLOptions{User: "stalwart", Database: "mail", Host: "db", Port: "5432", OutPath: "/tmp/dump.sql"})
joined := strings.Join(args, " ")
if !strings.Contains(joined, "-v ON_ERROR_STOP=1") {
t.Errorf("psql args %q must set ON_ERROR_STOP=1", joined)
}
for _, want := range []string{"-U stalwart", "-d mail", "-h db", "-p 5432", "-f /tmp/dump.sql"} {
if !strings.Contains(joined, want) {
t.Errorf("psql args %q missing %q", joined, want)
}
}
}
func TestBuildMySQLRestoreArgsOmitsUnsetConnectionFields(t *testing.T) {
args := BuildMySQLRestoreArgs(backup.SQLOptions{User: "stalwart", Database: "mail"})
if got, want := strings.Join(args, " "), "-u stalwart mail"; got != want {
t.Errorf("mysql args = %q, want %q", got, want)
}
}
func TestRunPsqlRestorePassesPasswordViaEnvironmentNotArgv(t *testing.T) {
dir := t.TempDir()
log := argsFile(t, dir)
withFakeExecutable(t, "psql", fakeScriptLoggingArgs(log, "echo \"PGPASSWORD=$PGPASSWORD\" >> "+log))
err := RunPsqlRestore(context.Background(), backup.SQLOptions{
User: "stalwart", Database: "mail", Password: "hunter2", OutPath: filepath.Join(dir, "dump.sql"),
})
if err != nil {
t.Fatal(err)
}
logged := readFile(t, log)
if !strings.Contains(logged, "PGPASSWORD=hunter2") {
t.Errorf("psql invocation %q should receive the password via PGPASSWORD", logged)
}
argv := strings.SplitN(logged, "\n", 2)[0]
if strings.Contains(argv, "hunter2") {
t.Errorf("psql argv %q contains the password - anything running `ps` could read it", argv)
}
}
func TestRunPsqlRestoreSurfacesFailureOutput(t *testing.T) {
dir := t.TempDir()
withFakeExecutable(t, "psql", "#!/bin/sh\necho 'ERROR: relation \"s\" already exists' >&2\nexit 1\n")
err := RunPsqlRestore(context.Background(), backup.SQLOptions{User: "u", Database: "d", OutPath: filepath.Join(dir, "dump.sql")})
if err == nil {
t.Fatal("want error when psql fails, got nil")
}
if !strings.Contains(err.Error(), "already exists") {
t.Errorf("error %q should carry psql's own output", err)
}
}
func TestRunMySQLRestoreFeedsTheDumpOnStdin(t *testing.T) {
dir := t.TempDir()
log := argsFile(t, dir)
dump := filepath.Join(dir, "dump.sql")
if err := os.WriteFile(dump, []byte("INSERT INTO s VALUES (1);\n"), 0o640); err != nil {
t.Fatal(err)
}
withFakeExecutable(t, "mysql", fakeScriptLoggingArgs(log, "cat >> "+log))
err := RunMySQLRestore(context.Background(), backup.SQLOptions{
User: "stalwart", Database: "mail", Password: "hunter2", OutPath: dump,
})
if err != nil {
t.Fatal(err)
}
if got := readFile(t, log); !strings.Contains(got, "INSERT INTO s VALUES (1);") {
t.Errorf("mysql stdin = %q, want the dump's contents", got)
}
}
func TestRunMySQLRestoreFailsOnMissingDump(t *testing.T) {
withFakeExecutable(t, "mysql", "#!/bin/sh\nexit 0\n")
err := RunMySQLRestore(context.Background(), backup.SQLOptions{User: "u", Database: "d", OutPath: filepath.Join(t.TempDir(), "absent.sql")})
if err == nil {
t.Fatal("want error when the dump file doesn't exist, got nil")
}
}