Initial commit: stalwart-migrator design and scaffolding
In-place upgrade tool for Stalwart Mail Server (0.15.5 -> latest) with checkpointed rollback and post-migration validation. Design stage; see ARCHITECTURE.md.
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/johnellis/stalwart-migrator/internal/checkpoint"
|
||||
)
|
||||
|
||||
// Options configures a full backup pass. Which fields matter depends on
|
||||
// rs.Topology.StoreBackend, as recorded by the preflight phase - Run
|
||||
// branches on that rather than requiring the caller to pre-select a code
|
||||
// path.
|
||||
type Options struct {
|
||||
// Binary preservation.
|
||||
BinaryPath string
|
||||
// SkipBinaryPreservation, when true, never touches BinaryPath - used by
|
||||
// a dry run, which must not move the production binary aside just to
|
||||
// simulate a migration it hasn't committed to.
|
||||
SkipBinaryPreservation bool
|
||||
|
||||
// Embedded backend (RocksDB/SQLite).
|
||||
DataDir string
|
||||
BackupDir string // destination for the fs snapshot; required if the backend is embedded
|
||||
|
||||
// External SQL backend (PostgreSQL/MySQL).
|
||||
SQL SQLOptions
|
||||
|
||||
// FoundationDB backend.
|
||||
FDB FDBOptions
|
||||
|
||||
// Settings/principals dump (always runs - every topology needs it).
|
||||
MigrationScriptURL string // defaults to DefaultMigrationScriptURL
|
||||
MigrationScriptSHA256 string // pinned hash; empty accepts and reports whatever is fetched (see DownloadFile)
|
||||
ScriptDestPath string
|
||||
AdminURL string
|
||||
AdminUser string
|
||||
AdminPassword string
|
||||
SettingsDumpPath string
|
||||
PrincipalsDumpPath string
|
||||
PythonPath string
|
||||
HTTPClient *http.Client
|
||||
|
||||
// Per-account content export (Vandelay) - optional defense-in-depth
|
||||
// layer. Empty Accounts skips it entirely; this is expected until
|
||||
// account enumeration is wired up (see stalwartapi.Client.AccountSnapshot).
|
||||
Vandelay VandelayOptions
|
||||
Accounts []string
|
||||
}
|
||||
|
||||
// Run executes the full backup pass described in ARCHITECTURE.md §4.2,
|
||||
// checkpointing each step. Unlike preflight, most steps here are hard
|
||||
// failures: a backup that didn't actually happen must stop the pipeline,
|
||||
// not just get reported and continued past. The one genuinely optional
|
||||
// layer is the per-account Vandelay export, which is skipped (Status:
|
||||
// StatusSkipped, not a failure) when Options.Accounts is empty - but if the
|
||||
// operator populated Accounts, a failure there is hard too, since silently
|
||||
// downgrading an explicitly requested backup layer would be worse than
|
||||
// stopping.
|
||||
func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) {
|
||||
var report Report
|
||||
|
||||
step := func(name string, fn func() (checkpoint.StepOutcome, error)) (checkpoint.StepOutcome, error) {
|
||||
outcome, err := store.RunStep(rs, checkpoint.PhaseBackup, name, fn)
|
||||
if err != nil {
|
||||
report.Results = append(report.Results, CheckResult{Name: name, Status: StatusFail, Detail: err.Error()})
|
||||
return outcome, err
|
||||
}
|
||||
status := Status(outcome.Verdict)
|
||||
if status == "" {
|
||||
status = StatusOK
|
||||
}
|
||||
report.Results = append(report.Results, CheckResult{Name: name, Status: status, Detail: outcome.Detail})
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
if opts.SkipBinaryPreservation {
|
||||
report.Results = append(report.Results, CheckResult{
|
||||
Name: "preserve-binary", Status: StatusSkipped,
|
||||
Detail: "skipped (SkipBinaryPreservation) - the production binary is never touched, e.g. for a dry run",
|
||||
})
|
||||
} else if _, err := step("preserve-binary", func() (checkpoint.StepOutcome, error) {
|
||||
preserved, err := PreserveBinary(opts.BinaryPath, rs.SourceVersion)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, size, err := hashFile(preserved)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("backup: hash preserved binary %s: %w", preserved, err)
|
||||
}
|
||||
rs.RecordArtifact("old-binary", checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size})
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("preserved %s as %s", opts.BinaryPath, preserved), Extra: preserved}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
backends := strings.ToLower(rs.Topology.StoreBackend)
|
||||
|
||||
switch {
|
||||
case strings.Contains(backends, "rocksdb") || strings.Contains(backends, "sqlite"):
|
||||
if _, err := step("fs-snapshot", func() (checkpoint.StepOutcome, error) {
|
||||
manifest, err := CopyDataDir(opts.DataDir, opts.BackupDir)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
manifestPath := filepath.Join(opts.BackupDir, "..", filepath.Base(opts.BackupDir)+".manifest.json")
|
||||
if err := WriteManifest(manifestPath, manifest); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, err := manifest.Checksum()
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact("fs-backup", checkpoint.Artifact{Path: opts.BackupDir, SHA256: sum, SizeBytes: manifest.TotalBytes})
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("copied %d file(s), %d bytes, from %s to %s", len(manifest.Files), manifest.TotalBytes, opts.DataDir, opts.BackupDir),
|
||||
Extra: manifestPath,
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if _, err := step("fs-verify", func() (checkpoint.StepOutcome, error) {
|
||||
manifestOutcome := rs.Outcome(checkpoint.PhaseBackup, "fs-snapshot")
|
||||
manifest, err := ReadManifest(manifestOutcome.Extra)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
if err := VerifyDataDirBackup(opts.BackupDir, manifest); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("re-hashed %d file(s), all match the manifest recorded at copy time", len(manifest.Files))}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
case strings.Contains(backends, "postgresql"):
|
||||
if _, err := step("sql-dump", func() (checkpoint.StepOutcome, error) {
|
||||
if err := RunPgDump(ctx, opts.SQL); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, size, err := hashFile(opts.SQL.OutPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact("sql-dump", checkpoint.Artifact{Path: opts.SQL.OutPath, SHA256: sum, SizeBytes: size})
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("pg_dump of critical tables (%s) to %s, %d bytes", strings.Join(criticalTables, " "), opts.SQL.OutPath, size)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
case strings.Contains(backends, "mysql"):
|
||||
if _, err := step("sql-dump", func() (checkpoint.StepOutcome, error) {
|
||||
if err := RunMySQLDump(ctx, opts.SQL); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
sum, size, err := hashFile(opts.SQL.OutPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact("sql-dump", checkpoint.Artifact{Path: opts.SQL.OutPath, SHA256: sum, SizeBytes: size})
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("mysqldump of critical tables (%s) to %s, %d bytes", strings.Join(criticalTables, " "), opts.SQL.OutPath, size)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
case strings.Contains(backends, "foundationdb"):
|
||||
if _, err := step("fdb-backup", func() (checkpoint.StepOutcome, error) {
|
||||
if err := StartFDBBackup(ctx, opts.FDB); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("fdbbackup start issued for destination %s (tag %s) - this only confirms the job was accepted, not that it finished; check `fdbbackup status` before relying on it", opts.FDB.Destination, opts.FDB.Tag),
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
default:
|
||||
report.Results = append(report.Results, CheckResult{
|
||||
Name: "backend-backup", Status: StatusSkipped,
|
||||
Detail: fmt.Sprintf("no known store backend recorded for this run (topology.store_backend=%q) - preflight must run first, or the backend wasn't recognized; no filesystem/DB backup was taken", rs.Topology.StoreBackend),
|
||||
})
|
||||
}
|
||||
|
||||
if _, err := step("settings-dump", func() (checkpoint.StepOutcome, error) {
|
||||
scriptURL := opts.MigrationScriptURL
|
||||
if scriptURL == "" {
|
||||
scriptURL = DefaultMigrationScriptURL
|
||||
}
|
||||
sum, err := DownloadFile(ctx, opts.HTTPClient, scriptURL, opts.ScriptDestPath, opts.MigrationScriptSHA256)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
pinNote := ""
|
||||
if opts.MigrationScriptSHA256 == "" {
|
||||
pinNote = fmt.Sprintf(" (no pin was configured - record sha256 %s as MigrationScriptSHA256 to pin it for future runs)", sum)
|
||||
}
|
||||
if err := RunSettingsDump(ctx, SettingsDumpOptions{
|
||||
PythonPath: opts.PythonPath,
|
||||
ScriptPath: opts.ScriptDestPath,
|
||||
URL: opts.AdminURL,
|
||||
Username: opts.AdminUser,
|
||||
Password: opts.AdminPassword,
|
||||
SettingsPath: opts.SettingsDumpPath,
|
||||
PrincipalsPath: opts.PrincipalsDumpPath,
|
||||
}); err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
settingsSum, settingsSize, err := hashFile(opts.SettingsDumpPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
principalsSum, principalsSize, err := hashFile(opts.PrincipalsDumpPath)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact("settings-dump", checkpoint.Artifact{Path: opts.SettingsDumpPath, SHA256: settingsSum, SizeBytes: settingsSize})
|
||||
rs.RecordArtifact("principals-dump", checkpoint.Artifact{Path: opts.PrincipalsDumpPath, SHA256: principalsSum, SizeBytes: principalsSize})
|
||||
return checkpoint.StepOutcome{
|
||||
Detail: fmt.Sprintf("dumped settings (%d bytes) and principals (%d bytes) from %s%s", settingsSize, principalsSize, opts.AdminURL, pinNote),
|
||||
Extra: sum,
|
||||
}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if len(opts.Accounts) == 0 {
|
||||
report.Results = append(report.Results, CheckResult{
|
||||
Name: "vandelay-export", Status: StatusSkipped,
|
||||
Detail: "no account list supplied - skipped; pass Options.Accounts (or --full-content-backup with an account source once account enumeration is wired up) to enable this belt-and-suspenders layer",
|
||||
})
|
||||
} else if _, err := step("vandelay-export", func() (checkpoint.StepOutcome, error) {
|
||||
if err := os.MkdirAll(opts.Vandelay.OutDir, 0o750); err != nil {
|
||||
return checkpoint.StepOutcome{}, fmt.Errorf("backup: create vandelay output dir %s: %w", opts.Vandelay.OutDir, err)
|
||||
}
|
||||
files, err := ExportAccounts(ctx, opts.Vandelay, opts.Accounts)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
var totalSize int64
|
||||
for i, f := range files {
|
||||
sum, size, err := hashFile(f)
|
||||
if err != nil {
|
||||
return checkpoint.StepOutcome{}, err
|
||||
}
|
||||
rs.RecordArtifact(fmt.Sprintf("vandelay-%s", opts.Accounts[i]), checkpoint.Artifact{Path: f, SHA256: sum, SizeBytes: size})
|
||||
totalSize += size
|
||||
}
|
||||
return checkpoint.StepOutcome{Detail: fmt.Sprintf("exported %d account(s), %d bytes total, to %s", len(files), totalSize, opts.Vandelay.OutDir)}, nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/johnellis/stalwart-migrator/internal/checkpoint"
|
||||
)
|
||||
|
||||
func TestBackupRunEndToEndAndResume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(dir, "data")
|
||||
if err := os.MkdirAll(dataDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("some rocksdb-shaped data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backupDir := filepath.Join(dir, "data-backup")
|
||||
|
||||
settingsPath := filepath.Join(dir, "settings.json")
|
||||
principalsPath := filepath.Join(dir, "principals.json")
|
||||
pythonScript := fmt.Sprintf("#!/bin/sh\necho fake-settings > %q\necho fake-principals > %q\n", settingsPath, principalsPath)
|
||||
pythonDir := withFakeExecutable(t, "python3", pythonScript)
|
||||
pythonScriptPath := filepath.Join(pythonDir, "python3")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("print('fake migrate_v016.py')\n"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
store := checkpoint.NewStore(t.TempDir())
|
||||
rs, err := store.Create("0.15.5", "0.16.14")
|
||||
if err != nil {
|
||||
t.Fatalf("store.Create: %v", err)
|
||||
}
|
||||
rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"}
|
||||
|
||||
opts := Options{
|
||||
BinaryPath: binaryPath,
|
||||
DataDir: dataDir,
|
||||
BackupDir: backupDir,
|
||||
MigrationScriptURL: srv.URL,
|
||||
ScriptDestPath: filepath.Join(dir, "migrate_v016.py"),
|
||||
AdminURL: "https://mail.example.com",
|
||||
AdminUser: "admin",
|
||||
AdminPassword: "hunter2",
|
||||
SettingsDumpPath: settingsPath,
|
||||
PrincipalsDumpPath: principalsPath,
|
||||
}
|
||||
|
||||
report, err := Run(context.Background(), store, rs, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run #1: %v", err)
|
||||
}
|
||||
for _, r := range report.Results {
|
||||
if r.Status == StatusFail {
|
||||
t.Errorf("Run #1: step %s failed: %s", r.Name, r.Detail)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"old-binary", "fs-backup", "settings-dump", "principals-dump"} {
|
||||
if _, ok := rs.Artifacts[name]; !ok {
|
||||
t.Errorf("Run #1: expected artifact %q to be recorded, got %v", name, rs.Artifacts)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(binaryPath + ".v0.15.5"); err != nil {
|
||||
t.Errorf("preserved binary missing: %v", err)
|
||||
}
|
||||
|
||||
// -- simulate a crash and resume, making every non-idempotent step's
|
||||
// -- inputs unusable so a re-execution (rather than a skip) fails loudly.
|
||||
if err := os.RemoveAll(dataDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.Close()
|
||||
if err := os.WriteFile(pythonScriptPath, []byte("#!/bin/sh\necho should-not-run-again >&2\nexit 1\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resumed, err := store.Load(rs.RunID)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Load (resume): %v", err)
|
||||
}
|
||||
report2, err := Run(context.Background(), store, resumed, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run #2 (resume) should succeed without redoing completed steps: %v", err)
|
||||
}
|
||||
if len(report2.Results) != len(report.Results) {
|
||||
t.Errorf("resumed report has %d results, want %d (same as first run)", len(report2.Results), len(report.Results))
|
||||
}
|
||||
for _, r := range report2.Results {
|
||||
if r.Status == StatusFail {
|
||||
t.Errorf("Run #2 (resume): step %s unexpectedly failed: %s", r.Name, r.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRunSkipsVandelayWhenNoAccountsGiven(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
os.WriteFile(binaryPath, []byte("fake binary"), 0o755)
|
||||
dataDir := filepath.Join(dir, "data")
|
||||
os.MkdirAll(dataDir, 0o755)
|
||||
os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("x"), 0o644)
|
||||
|
||||
withFakeExecutable(t, "python3", fmt.Sprintf(
|
||||
"#!/bin/sh\necho x > %q\necho x > %q\n",
|
||||
filepath.Join(dir, "settings.json"), filepath.Join(dir, "principals.json"),
|
||||
))
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("script"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
store := checkpoint.NewStore(t.TempDir())
|
||||
rs, err := store.Create("0.15.5", "0.16.14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"}
|
||||
|
||||
opts := Options{
|
||||
BinaryPath: binaryPath,
|
||||
DataDir: dataDir,
|
||||
BackupDir: filepath.Join(dir, "data-backup"),
|
||||
MigrationScriptURL: srv.URL,
|
||||
ScriptDestPath: filepath.Join(dir, "migrate_v016.py"),
|
||||
AdminURL: "https://mail.example.com",
|
||||
SettingsDumpPath: filepath.Join(dir, "settings.json"),
|
||||
PrincipalsDumpPath: filepath.Join(dir, "principals.json"),
|
||||
// Accounts intentionally left empty.
|
||||
}
|
||||
|
||||
report, err := Run(context.Background(), store, rs, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range report.Results {
|
||||
if r.Name == "vandelay-export" {
|
||||
found = true
|
||||
if r.Status != StatusSkipped {
|
||||
t.Errorf("vandelay-export status = %s, want skipped", r.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("report missing a vandelay-export entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRunSkipsBinaryPreservationForDryRun(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dataDir := filepath.Join(dir, "data")
|
||||
os.MkdirAll(dataDir, 0o755)
|
||||
os.WriteFile(filepath.Join(dataDir, "db.bin"), []byte("x"), 0o644)
|
||||
|
||||
withFakeExecutable(t, "python3", fmt.Sprintf(
|
||||
"#!/bin/sh\necho x > %q\necho x > %q\n",
|
||||
filepath.Join(dir, "settings.json"), filepath.Join(dir, "principals.json"),
|
||||
))
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("script"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
store := checkpoint.NewStore(t.TempDir())
|
||||
rs, err := store.Create("0.15.5", "0.16.14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rs.Topology = checkpoint.Topology{StoreBackend: "rocksdb"}
|
||||
|
||||
opts := Options{
|
||||
BinaryPath: binaryPath,
|
||||
SkipBinaryPreservation: true,
|
||||
DataDir: dataDir,
|
||||
BackupDir: filepath.Join(dir, "data-backup"),
|
||||
MigrationScriptURL: srv.URL,
|
||||
ScriptDestPath: filepath.Join(dir, "migrate_v016.py"),
|
||||
AdminURL: "https://mail.example.com",
|
||||
SettingsDumpPath: filepath.Join(dir, "settings.json"),
|
||||
PrincipalsDumpPath: filepath.Join(dir, "principals.json"),
|
||||
}
|
||||
|
||||
report, err := Run(context.Background(), store, rs, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
for _, r := range report.Results {
|
||||
if r.Name == "preserve-binary" && r.Status != StatusSkipped {
|
||||
t.Errorf("preserve-binary status = %s, want skipped", r.Status)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(binaryPath); err != nil {
|
||||
t.Errorf("production binary at %s should be untouched, but stat failed: %v", binaryPath, err)
|
||||
}
|
||||
if _, ok := rs.Artifacts["old-binary"]; ok {
|
||||
t.Error("no old-binary artifact should be recorded when preservation is skipped")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// PreserveBinary moves the currently-installed binary aside to
|
||||
// "<binaryPath>.v<sourceVersion>" so rollback can restart the exact old
|
||||
// binary without re-downloading anything, and cutover can install the new
|
||||
// one at the original path. It never deletes the old binary, and it's
|
||||
// idempotent: if a prior attempt at this run already preserved it, calling
|
||||
// this again just returns the existing preserved path rather than erroring
|
||||
// on a missing source file.
|
||||
func PreserveBinary(binaryPath, sourceVersion string) (preservedPath string, err error) {
|
||||
if sourceVersion == "" {
|
||||
return "", fmt.Errorf("backup: cannot preserve %s without a source version to suffix it with", binaryPath)
|
||||
}
|
||||
preservedPath = binaryPath + ".v" + sourceVersion
|
||||
|
||||
if _, statErr := os.Stat(preservedPath); statErr == nil {
|
||||
return preservedPath, nil
|
||||
} else if !os.IsNotExist(statErr) {
|
||||
return "", fmt.Errorf("backup: stat %s: %w", preservedPath, statErr)
|
||||
}
|
||||
|
||||
if err := os.Rename(binaryPath, preservedPath); err != nil {
|
||||
return "", fmt.Errorf("backup: preserve %s as %s: %w", binaryPath, preservedPath, err)
|
||||
}
|
||||
return preservedPath, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreserveBinary(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
preserved, err := PreserveBinary(binaryPath, "0.15.5")
|
||||
if err != nil {
|
||||
t.Fatalf("PreserveBinary: %v", err)
|
||||
}
|
||||
wantPath := binaryPath + ".v0.15.5"
|
||||
if preserved != wantPath {
|
||||
t.Errorf("preserved = %s, want %s", preserved, wantPath)
|
||||
}
|
||||
if _, err := os.Stat(binaryPath); !os.IsNotExist(err) {
|
||||
t.Error("original binary path should no longer exist after preservation")
|
||||
}
|
||||
if _, err := os.Stat(preserved); err != nil {
|
||||
t.Errorf("preserved binary missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveBinaryIsIdempotent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
if err := os.WriteFile(binaryPath, []byte("fake binary"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
first, err := PreserveBinary(binaryPath, "0.15.5")
|
||||
if err != nil {
|
||||
t.Fatalf("PreserveBinary #1: %v", err)
|
||||
}
|
||||
|
||||
// Simulate a resumed run: the binary is already gone from binaryPath
|
||||
// (moved on the prior attempt). Calling again must not error just
|
||||
// because the source no longer exists.
|
||||
second, err := PreserveBinary(binaryPath, "0.15.5")
|
||||
if err != nil {
|
||||
t.Fatalf("PreserveBinary #2 (resume): %v", err)
|
||||
}
|
||||
if first != second {
|
||||
t.Errorf("preserved paths differ across calls: %s vs %s", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveBinaryRequiresSourceVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binaryPath := filepath.Join(dir, "stalwart")
|
||||
os.WriteFile(binaryPath, []byte("x"), 0o755)
|
||||
if _, err := PreserveBinary(binaryPath, ""); err == nil {
|
||||
t.Fatal("PreserveBinary with an empty source version should error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package backup implements the defense-in-depth backup: filesystem/DB snapshot, settings dump, and Vandelay content export.
|
||||
// See ARCHITECTURE.md §4.2 for the design.
|
||||
package backup
|
||||
@@ -0,0 +1,48 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// withFakeExecutable puts a fake executable named `name` at the front of
|
||||
// PATH for the duration of the test, so code that shells out to a
|
||||
// real-world tool (pg_dump, mysqldump, fdbbackup, vandelay, python3) can be
|
||||
// exercised without that tool actually being installed. t.Setenv restores
|
||||
// PATH automatically and marks the test non-parallel.
|
||||
func withFakeExecutable(t *testing.T, name, script string) (dir string) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
return dir
|
||||
}
|
||||
|
||||
// argsFile is a shared convention the fake scripts below use: they append
|
||||
// their own argv (space-joined, one invocation per line) to a file so the
|
||||
// test can assert on exactly what was passed.
|
||||
func argsFile(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
return filepath.Join(dir, "invoked-args.log")
|
||||
}
|
||||
|
||||
func readArgsFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func fakeScriptLoggingArgs(logPath string, extraBody string) string {
|
||||
return fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\n%s\n", logPath, extraBody)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// FDBOptions configures a FoundationDB backup via fdbbackup, FDB's own
|
||||
// backup CLI (not something Stalwart-specific) - see ARCHITECTURE.md §4.2.
|
||||
type FDBOptions struct {
|
||||
ClusterFile string // -C; empty uses fdbbackup's own default cluster file
|
||||
Destination string // -d, a backup URL e.g. "file:///var/backups/stalwart-fdb"
|
||||
Tag string // -t; defaults to "default" if empty
|
||||
}
|
||||
|
||||
func (o FDBOptions) tag() string {
|
||||
if o.Tag == "" {
|
||||
return "default"
|
||||
}
|
||||
return o.Tag
|
||||
}
|
||||
|
||||
// BuildFDBBackupStartArgs returns the fdbbackup argv for starting a backup,
|
||||
// without the leading "fdbbackup".
|
||||
func BuildFDBBackupStartArgs(o FDBOptions) []string {
|
||||
args := []string{"start"}
|
||||
if o.ClusterFile != "" {
|
||||
args = append(args, "-C", o.ClusterFile)
|
||||
}
|
||||
args = append(args, "-d", o.Destination, "-t", o.tag())
|
||||
return args
|
||||
}
|
||||
|
||||
// StartFDBBackup kicks off an fdbbackup run. fdbbackup start returns as soon
|
||||
// as the backup job is registered, not when it finishes - callers that need
|
||||
// to know it's done should poll FDBBackupStatus and look for their own
|
||||
// definition of "complete" in its output, since this wrapper doesn't parse
|
||||
// fdbbackup's status text (that format isn't stable enough here to depend
|
||||
// on without verifying it against the fdbbackup version actually in use).
|
||||
func StartFDBBackup(ctx context.Context, o FDBOptions) error {
|
||||
cmd := exec.CommandContext(ctx, "fdbbackup", BuildFDBBackupStartArgs(o)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: fdbbackup start failed: %w (output: %s)", err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FDBBackupStatus returns fdbbackup's raw status output for the given tag,
|
||||
// for a human or a caller-supplied parser to interpret.
|
||||
func FDBBackupStatus(ctx context.Context, o FDBOptions) (string, error) {
|
||||
args := []string{"status", "-t", o.tag()}
|
||||
if o.ClusterFile != "" {
|
||||
args = append(args, "-C", o.ClusterFile)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "fdbbackup", args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("backup: fdbbackup status failed: %w (output: %s)", err, out)
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildFDBBackupStartArgs(t *testing.T) {
|
||||
args := BuildFDBBackupStartArgs(FDBOptions{ClusterFile: "/etc/foundationdb/fdb.cluster", Destination: "file:///var/backups/stalwart-fdb", Tag: "stalwart-migrate"})
|
||||
joined := strings.Join(args, " ")
|
||||
for _, want := range []string{"start", "-C /etc/foundationdb/fdb.cluster", "-d file:///var/backups/stalwart-fdb", "-t stalwart-migrate"} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("BuildFDBBackupStartArgs = %q, missing %q", joined, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFDBBackupStartArgsDefaultTag(t *testing.T) {
|
||||
args := BuildFDBBackupStartArgs(FDBOptions{Destination: "file:///var/backups/stalwart-fdb"})
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "-t default") {
|
||||
t.Errorf("BuildFDBBackupStartArgs = %q, want default tag when none given", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartFDBBackupInvokesFdbbackup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
withFakeExecutable(t, "fdbbackup", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
err := StartFDBBackup(context.Background(), FDBOptions{Destination: "file:///var/backups/stalwart-fdb"})
|
||||
if err != nil {
|
||||
t.Fatalf("StartFDBBackup: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
if !strings.Contains(got, "start") || !strings.Contains(got, "-d file:///var/backups/stalwart-fdb") {
|
||||
t.Errorf("fdbbackup was invoked with %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartFDBBackupPropagatesFailure(t *testing.T) {
|
||||
withFakeExecutable(t, "fdbbackup", "#!/bin/sh\necho 'cluster unreachable' >&2\nexit 1\n")
|
||||
err := StartFDBBackup(context.Background(), FDBOptions{Destination: "file:///var/backups/stalwart-fdb"})
|
||||
if err == nil {
|
||||
t.Fatal("StartFDBBackup should error when fdbbackup exits non-zero")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Manifest records the per-file checksums produced by CopyDataDir, so a
|
||||
// later Verify pass can detect corruption or truncation introduced by the
|
||||
// copy itself. It is NOT authentication that the copied store is a valid,
|
||||
// openable Stalwart database - confirming that would mean booting the old
|
||||
// binary read-only against the backup, which needs config/CLI details this
|
||||
// tool doesn't verify yet (see the parallel caveat on
|
||||
// stalwartapi.Client.AccountSnapshot). Treat a clean Verify as "the bytes we
|
||||
// wrote match the bytes we copied", not "Stalwart can open this".
|
||||
type Manifest struct {
|
||||
SourceDir string `json:"source_dir"`
|
||||
Files []ManifestEntry `json:"files"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
}
|
||||
|
||||
type ManifestEntry struct {
|
||||
RelPath string `json:"rel_path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Checksum returns a single content hash identifying this manifest (and
|
||||
// transitively, the exact set of files and bytes it describes), suitable
|
||||
// for recording as a checkpoint.Artifact's SHA256 - a directory doesn't
|
||||
// have one natural hash, so this stands in for it.
|
||||
func (m *Manifest) Checksum() (string, error) {
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup: marshal manifest: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// WriteManifest persists a manifest as JSON next to the backup it describes.
|
||||
func WriteManifest(path string, m *Manifest) error {
|
||||
data, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: marshal manifest: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o640); err != nil {
|
||||
return fmt.Errorf("backup: write manifest %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadManifest loads a manifest previously written by WriteManifest.
|
||||
func ReadManifest(path string) (*Manifest, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read manifest %s: %w", path, err)
|
||||
}
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return nil, fmt.Errorf("backup: parse manifest %s: %w", path, err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// CopyDataDir copies srcDir to dstDir the way `cp -a` would (directories,
|
||||
// regular files, and symlinks, preserving regular-file permissions),
|
||||
// hashing every regular file as it's written. dstDir is cleared first if it
|
||||
// already exists, so a retried step after a partial failure produces a
|
||||
// clean copy rather than a mix of old and new files - filesystem copies
|
||||
// aren't resumable at the byte level in any way worth building here, so a
|
||||
// retry just redoes the whole thing.
|
||||
func CopyDataDir(srcDir, dstDir string) (*Manifest, error) {
|
||||
srcAbs, err := filepath.Abs(srcDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: resolve source %s: %w", srcDir, err)
|
||||
}
|
||||
dstAbs, err := filepath.Abs(dstDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: resolve destination %s: %w", dstDir, err)
|
||||
}
|
||||
if srcAbs == dstAbs {
|
||||
return nil, fmt.Errorf("backup: source and destination are the same path: %s", srcAbs)
|
||||
}
|
||||
if strings.HasPrefix(dstAbs+string(filepath.Separator), srcAbs+string(filepath.Separator)) {
|
||||
return nil, fmt.Errorf("backup: destination %s is inside source %s - refusing to copy a directory into itself", dstAbs, srcAbs)
|
||||
}
|
||||
if err := os.RemoveAll(dstAbs); err != nil {
|
||||
return nil, fmt.Errorf("backup: clear stale destination %s: %w", dstAbs, err)
|
||||
}
|
||||
if err := os.MkdirAll(dstAbs, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("backup: create destination %s: %w", dstAbs, err)
|
||||
}
|
||||
|
||||
manifest := &Manifest{SourceDir: srcAbs}
|
||||
walkErr := filepath.WalkDir(srcAbs, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(srcAbs, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
dstPath := filepath.Join(dstAbs, rel)
|
||||
|
||||
if d.Type()&fs.ModeSymlink != 0 {
|
||||
target, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readlink %s: %w", path, err)
|
||||
}
|
||||
return os.Symlink(target, dstPath)
|
||||
}
|
||||
if d.IsDir() {
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(dstPath, info.Mode().Perm())
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("backup: unsupported file type at %s (mode %s)", path, info.Mode())
|
||||
}
|
||||
sum, size, err := copyFileWithChecksum(path, dstPath, info.Mode().Perm())
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy %s: %w", path, err)
|
||||
}
|
||||
manifest.Files = append(manifest.Files, ManifestEntry{RelPath: rel, SHA256: sum, Size: size})
|
||||
manifest.TotalBytes += size
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
return nil, fmt.Errorf("backup: copy %s to %s: %w", srcAbs, dstAbs, walkErr)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func copyFileWithChecksum(src, dst string, perm fs.FileMode) (sha256Hex string, size int64, err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer out.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(out, h), in)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
|
||||
// VerifyDataDirBackup recomputes every file's checksum under dstDir and
|
||||
// compares it against the manifest CopyDataDir produced, returning a single
|
||||
// error describing every mismatch found (not just the first) so a human
|
||||
// sees the full extent of any corruption in one pass.
|
||||
func VerifyDataDirBackup(dstDir string, m *Manifest) error {
|
||||
var problems []string
|
||||
for _, entry := range m.Files {
|
||||
path := filepath.Join(dstDir, entry.RelPath)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s: %v", entry.RelPath, err))
|
||||
continue
|
||||
}
|
||||
h := sha256.New()
|
||||
size, err := io.Copy(h, f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s: read error: %v", entry.RelPath, err))
|
||||
continue
|
||||
}
|
||||
if size != entry.Size {
|
||||
problems = append(problems, fmt.Sprintf("%s: size %d, want %d", entry.RelPath, size, entry.Size))
|
||||
continue
|
||||
}
|
||||
if got := hex.EncodeToString(h.Sum(nil)); got != entry.SHA256 {
|
||||
problems = append(problems, fmt.Sprintf("%s: sha256 %s, want %s", entry.RelPath, got, entry.SHA256))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("backup: verification failed for %d of %d file(s):\n%s", len(problems), len(m.Files), strings.Join(problems, "\n"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeTree(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "a.db"), []byte("alpha data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "sub", "b.db"), []byte("bravo data, a bit longer"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("a.db", filepath.Join(root, "link-to-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyDataDirAndVerify(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "data")
|
||||
if err := os.MkdirAll(src, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTree(t, src)
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "data-backup")
|
||||
manifest, err := CopyDataDir(src, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("CopyDataDir: %v", err)
|
||||
}
|
||||
if len(manifest.Files) != 2 {
|
||||
t.Fatalf("manifest has %d files, want 2 (symlinks aren't hashed): %+v", len(manifest.Files), manifest.Files)
|
||||
}
|
||||
if manifest.TotalBytes != int64(len("alpha data")+len("bravo data, a bit longer")) {
|
||||
t.Errorf("TotalBytes = %d, want %d", manifest.TotalBytes, len("alpha data")+len("bravo data, a bit longer"))
|
||||
}
|
||||
|
||||
// The copy should be byte-identical, including the symlink.
|
||||
got, err := os.ReadFile(filepath.Join(dst, "sub", "b.db"))
|
||||
if err != nil || string(got) != "bravo data, a bit longer" {
|
||||
t.Errorf("copied sub/b.db = %q, %v", got, err)
|
||||
}
|
||||
target, err := os.Readlink(filepath.Join(dst, "link-to-a"))
|
||||
if err != nil || target != "a.db" {
|
||||
t.Errorf("copied symlink target = %q, %v, want a.db", target, err)
|
||||
}
|
||||
|
||||
if err := VerifyDataDirBackup(dst, manifest); err != nil {
|
||||
t.Errorf("VerifyDataDirBackup on an untouched copy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyDataDirRefusesSelfCopy(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "data")
|
||||
os.MkdirAll(dir, 0o755)
|
||||
if _, err := CopyDataDir(dir, dir); err == nil {
|
||||
t.Fatal("CopyDataDir(dir, dir) should refuse to copy a directory onto itself")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyDataDirRefusesNestedDestination(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "data")
|
||||
os.MkdirAll(src, 0o755)
|
||||
nested := filepath.Join(src, "backup")
|
||||
if _, err := CopyDataDir(src, nested); err == nil {
|
||||
t.Fatal("CopyDataDir should refuse a destination nested inside the source")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyDataDirRetryClearsStaleFiles(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "data")
|
||||
os.MkdirAll(src, 0o755)
|
||||
writeTree(t, src)
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "data-backup")
|
||||
// Simulate a stale partial copy from a previous failed attempt.
|
||||
os.MkdirAll(dst, 0o755)
|
||||
if err := os.WriteFile(filepath.Join(dst, "stale-leftover.tmp"), []byte("junk"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
manifest, err := CopyDataDir(src, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("CopyDataDir: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dst, "stale-leftover.tmp")); !os.IsNotExist(err) {
|
||||
t.Error("stale-leftover.tmp should have been cleared by a fresh copy, but still exists")
|
||||
}
|
||||
if err := VerifyDataDirBackup(dst, manifest); err != nil {
|
||||
t.Errorf("VerifyDataDirBackup after retry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDataDirBackupDetectsTampering(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "data")
|
||||
os.MkdirAll(src, 0o755)
|
||||
writeTree(t, src)
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "data-backup")
|
||||
manifest, err := CopyDataDir(src, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("CopyDataDir: %v", err)
|
||||
}
|
||||
|
||||
// Corrupt the backup after the fact - Verify must catch it.
|
||||
if err := os.WriteFile(filepath.Join(dst, "a.db"), []byte("corrupted!"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := VerifyDataDirBackup(dst, manifest); err == nil {
|
||||
t.Fatal("VerifyDataDirBackup should have detected the tampered file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestChecksumIsDeterministic(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "data")
|
||||
os.MkdirAll(src, 0o755)
|
||||
writeTree(t, src)
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "data-backup")
|
||||
m1, err := CopyDataDir(src, dst)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum1, err := m1.Checksum()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dst2 := filepath.Join(t.TempDir(), "data-backup-2")
|
||||
m2, err := CopyDataDir(src, dst2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum2, err := m2.Checksum()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if sum1 != sum2 {
|
||||
t.Errorf("two copies of the same source produced different manifest checksums: %s vs %s", sum1, sum2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReadManifestRoundtrip(t *testing.T) {
|
||||
m := &Manifest{
|
||||
SourceDir: "/var/lib/stalwart",
|
||||
Files: []ManifestEntry{{RelPath: "a.db", SHA256: "deadbeef", Size: 42}},
|
||||
TotalBytes: 42,
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "manifest.json")
|
||||
if err := WriteManifest(path, m); err != nil {
|
||||
t.Fatalf("WriteManifest: %v", err)
|
||||
}
|
||||
got, err := ReadManifest(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadManifest: %v", err)
|
||||
}
|
||||
if got.TotalBytes != 42 || len(got.Files) != 1 || got.Files[0].SHA256 != "deadbeef" {
|
||||
t.Errorf("roundtrip mismatch: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// hashFile returns a file's SHA256 and size, for recording as a
|
||||
// checkpoint.Artifact.
|
||||
func hashFile(path string) (sha256Hex string, size int64, err error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: hash %s: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: hash %s: %w", path, err)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Status is a single backup step's verdict. Unlike preflight, most backup
|
||||
// steps that fail are hard failures (Run aborts) rather than advisory - see
|
||||
// Run's doc comment - but Status still distinguishes "did the thing" from
|
||||
// "correctly skipped because it doesn't apply to this deployment".
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusOK Status = "ok"
|
||||
StatusSkipped Status = "skipped"
|
||||
StatusFail Status = "fail"
|
||||
)
|
||||
|
||||
type CheckResult struct {
|
||||
Name string
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Results []CheckResult
|
||||
}
|
||||
|
||||
func (r Report) String() string {
|
||||
var b strings.Builder
|
||||
for _, res := range r.Results {
|
||||
fmt.Fprintf(&b, "[%-7s] %-18s %s\n", strings.ToUpper(string(res.Status)), res.Name, res.Detail)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultMigrationScriptURL is Stalwart's own v0.15->v0.16 settings
|
||||
// converter, referenced directly from UPGRADING/v0_16.md. It's an external,
|
||||
// Stalwart-owned dependency this tool doesn't vendor a copy of - see the
|
||||
// pinning discussion on DownloadFile and ARCHITECTURE.md §8.
|
||||
const DefaultMigrationScriptURL = "https://raw.githubusercontent.com/stalwartlabs/stalwart/main/resources/scripts/migrate_v016.py"
|
||||
|
||||
// DownloadFile fetches url to destPath and returns its SHA256. If
|
||||
// expectedSHA256 is non-empty, a mismatching download is rejected (and the
|
||||
// partial file removed) - this is how a pinned migration-script hash is
|
||||
// enforced, so a run never silently executes a different version of a
|
||||
// script than the one it was reviewed against. If expectedSHA256 is empty,
|
||||
// the download is accepted unconditionally and its hash is returned so the
|
||||
// caller can record it as the pin for next time; first-run trust-on-first-use
|
||||
// is a known gap, flagged in ARCHITECTURE.md §8.
|
||||
func DownloadFile(ctx context.Context, httpClient *http.Client, url, destPath, expectedSHA256 string) (sha256Hex string, err error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 60 * time.Second}
|
||||
}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup: fetch %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("backup: fetch %s: unexpected status %s", url, resp.Status)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup: create %s: %w", destPath, err)
|
||||
}
|
||||
h := sha256.New()
|
||||
_, copyErr := io.Copy(io.MultiWriter(f, h), resp.Body)
|
||||
closeErr := f.Close()
|
||||
if copyErr != nil {
|
||||
os.Remove(destPath)
|
||||
return "", fmt.Errorf("backup: download %s: %w", url, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
os.Remove(destPath)
|
||||
return "", fmt.Errorf("backup: close %s: %w", destPath, closeErr)
|
||||
}
|
||||
|
||||
sha256Hex = hex.EncodeToString(h.Sum(nil))
|
||||
if expectedSHA256 != "" && sha256Hex != expectedSHA256 {
|
||||
os.Remove(destPath)
|
||||
return "", fmt.Errorf(
|
||||
"backup: %s checksum mismatch: got %s, want %s (refusing to run an unexpected version of a script that irreversibly wipes settings on first v0.16 start)",
|
||||
url, sha256Hex, expectedSHA256,
|
||||
)
|
||||
}
|
||||
return sha256Hex, nil
|
||||
}
|
||||
|
||||
// SettingsDumpOptions configures a migrate_v016.py `dump` invocation
|
||||
// against a live v0.15.x instance - see UPGRADING/v0_16.md.
|
||||
type SettingsDumpOptions struct {
|
||||
PythonPath string // defaults to "python3"
|
||||
ScriptPath string // local path to the already-downloaded, checksum-verified script
|
||||
URL string // the running v0.15.x instance's base URL
|
||||
Username string
|
||||
Password string
|
||||
SettingsPath string
|
||||
PrincipalsPath string
|
||||
}
|
||||
|
||||
// RunSettingsDump runs migrate_v016.py's dump subcommand, which reads the
|
||||
// live v0.15.x server's settings and principals over its admin API and
|
||||
// writes them to SettingsPath/PrincipalsPath for the later convert step
|
||||
// (ARCHITECTURE.md §4.3). This step is read-only against the server, so
|
||||
// it's safe to run well before cutover - ARCHITECTURE.md §4.2 calls for it
|
||||
// both at preflight time and again immediately before cutover, since the
|
||||
// live settings may have changed in between.
|
||||
func RunSettingsDump(ctx context.Context, o SettingsDumpOptions) error {
|
||||
python := o.PythonPath
|
||||
if python == "" {
|
||||
python = "python3"
|
||||
}
|
||||
args := []string{
|
||||
o.ScriptPath, "dump",
|
||||
"--url", o.URL,
|
||||
"--username", o.Username,
|
||||
"--password", o.Password,
|
||||
"--settings", o.SettingsPath,
|
||||
"--principals", o.PrincipalsPath,
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, python, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: migrate_v016.py dump failed: %w (output: %s)", err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SettingsConvertOptions configures a migrate_v016.py `convert` invocation,
|
||||
// which turns the settings/principals dump into the v0.16 config.json and
|
||||
// export.json that recovery mode consumes - see UPGRADING/v0_16.md.
|
||||
type SettingsConvertOptions struct {
|
||||
PythonPath string // defaults to "python3"
|
||||
ScriptPath string
|
||||
SettingsPath string
|
||||
PrincipalsPath string
|
||||
ConfigPath string // output: config.json for the new binary's --config flag
|
||||
OutputPath string // output: export.json for `stalwart-cli apply`
|
||||
|
||||
// PatchPaths rewrites path prefixes in the generated config (documented
|
||||
// for Docker deployments as "--patch-paths /opt/stalwart=/var/lib/stalwart",
|
||||
// e.g. old-path -> new-path). This is the officially documented
|
||||
// mechanism a dry-run relies on to point the generated config at a
|
||||
// sandbox data directory instead of the production one - see
|
||||
// ARCHITECTURE.md's dry-run design - rather than this tool editing
|
||||
// config.json's contents directly, which would require depending on its
|
||||
// exact schema.
|
||||
PatchPaths map[string]string
|
||||
}
|
||||
|
||||
// RunSettingsConvert runs migrate_v016.py's convert subcommand.
|
||||
func RunSettingsConvert(ctx context.Context, o SettingsConvertOptions) error {
|
||||
python := o.PythonPath
|
||||
if python == "" {
|
||||
python = "python3"
|
||||
}
|
||||
args := []string{
|
||||
o.ScriptPath, "convert",
|
||||
"--settings", o.SettingsPath,
|
||||
"--principals", o.PrincipalsPath,
|
||||
"--config", o.ConfigPath,
|
||||
"--output", o.OutputPath,
|
||||
}
|
||||
if len(o.PatchPaths) > 0 {
|
||||
pairs := make([]string, 0, len(o.PatchPaths))
|
||||
for old, new := range o.PatchPaths {
|
||||
pairs = append(pairs, old+"="+new)
|
||||
}
|
||||
sort.Strings(pairs) // deterministic argv, easier to test and to log
|
||||
args = append(args, "--patch-paths", strings.Join(pairs, ","))
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, python, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: migrate_v016.py convert failed: %w (output: %s)", err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloadFileAcceptsMatchingChecksum(t *testing.T) {
|
||||
content := "print('fake migration script')\n"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(content))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
expected := hex.EncodeToString(sum[:])
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "script.py")
|
||||
got, err := DownloadFile(context.Background(), nil, srv.URL, dest, expected)
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadFile: %v", err)
|
||||
}
|
||||
if got != expected {
|
||||
t.Errorf("returned checksum = %s, want %s", got, expected)
|
||||
}
|
||||
data, _ := os.ReadFile(dest)
|
||||
if string(data) != content {
|
||||
t.Errorf("downloaded content = %q, want %q", data, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileRejectsMismatchedChecksum(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("unexpected content"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "script.py")
|
||||
_, err := DownloadFile(context.Background(), nil, srv.URL, dest, "0000000000000000000000000000000000000000000000000000000000000000")
|
||||
if err == nil {
|
||||
t.Fatal("DownloadFile should reject a checksum mismatch")
|
||||
}
|
||||
if _, statErr := os.Stat(dest); !os.IsNotExist(statErr) {
|
||||
t.Error("DownloadFile should remove the file it wrote after a checksum mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileWithoutPinReturnsComputedHash(t *testing.T) {
|
||||
content := "arbitrary content"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(content))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "script.py")
|
||||
got, err := DownloadFile(context.Background(), nil, srv.URL, dest, "")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadFile: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
want := hex.EncodeToString(sum[:])
|
||||
if got != want {
|
||||
t.Errorf("returned checksum = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSettingsDumpInvokesScriptWithFlags(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
err := RunSettingsDump(context.Background(), SettingsDumpOptions{
|
||||
PythonPath: filepath.Join(pythonDir, "python3"),
|
||||
ScriptPath: "/opt/migrate_v016.py",
|
||||
URL: "https://mail.example.com",
|
||||
Username: "admin",
|
||||
Password: "hunter2",
|
||||
SettingsPath: filepath.Join(dir, "settings.json"),
|
||||
PrincipalsPath: filepath.Join(dir, "principals.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunSettingsDump: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
for _, want := range []string{"/opt/migrate_v016.py", "dump", "--url https://mail.example.com", "--username admin"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("script invoked with %q, missing %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSettingsDumpPropagatesFailure(t *testing.T) {
|
||||
pythonDir := withFakeExecutable(t, "python3", "#!/bin/sh\necho 'auth failed' >&2\nexit 1\n")
|
||||
err := RunSettingsDump(context.Background(), SettingsDumpOptions{
|
||||
PythonPath: filepath.Join(pythonDir, "python3"),
|
||||
ScriptPath: "/opt/migrate_v016.py",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("RunSettingsDump should error when the script exits non-zero")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "auth failed") {
|
||||
t.Errorf("error = %v, want it to include the script's stderr", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSettingsConvertInvokesScriptWithFlags(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
err := RunSettingsConvert(context.Background(), SettingsConvertOptions{
|
||||
PythonPath: filepath.Join(pythonDir, "python3"),
|
||||
ScriptPath: "/opt/migrate_v016.py",
|
||||
SettingsPath: filepath.Join(dir, "settings.json"),
|
||||
PrincipalsPath: filepath.Join(dir, "principals.json"),
|
||||
ConfigPath: filepath.Join(dir, "config.json"),
|
||||
OutputPath: filepath.Join(dir, "export.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunSettingsConvert: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
for _, want := range []string{"convert", "--settings", "--config", "--output"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("script invoked with %q, missing %q", got, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "--patch-paths") {
|
||||
t.Errorf("script invoked with %q, should not include --patch-paths when none given", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSettingsConvertWithPatchPaths(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
pythonDir := withFakeExecutable(t, "python3", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
err := RunSettingsConvert(context.Background(), SettingsConvertOptions{
|
||||
PythonPath: filepath.Join(pythonDir, "python3"),
|
||||
ScriptPath: "/opt/migrate_v016.py",
|
||||
SettingsPath: filepath.Join(dir, "settings.json"),
|
||||
PrincipalsPath: filepath.Join(dir, "principals.json"),
|
||||
ConfigPath: filepath.Join(dir, "config.json"),
|
||||
OutputPath: filepath.Join(dir, "export.json"),
|
||||
PatchPaths: map[string]string{"/var/lib/stalwart": "/tmp/sandbox/stalwart"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunSettingsConvert: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
if !strings.Contains(got, "--patch-paths /var/lib/stalwart=/tmp/sandbox/stalwart") {
|
||||
t.Errorf("script invoked with %q, missing the expected --patch-paths flag", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSettingsConvertPropagatesFailure(t *testing.T) {
|
||||
pythonDir := withFakeExecutable(t, "python3", "#!/bin/sh\necho 'unsupported settings key' >&2\nexit 1\n")
|
||||
err := RunSettingsConvert(context.Background(), SettingsConvertOptions{
|
||||
PythonPath: filepath.Join(pythonDir, "python3"),
|
||||
ScriptPath: "/opt/migrate_v016.py",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("RunSettingsConvert should error when the script exits non-zero")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported settings key") {
|
||||
t.Errorf("error = %v, want it to include the script's stderr", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// criticalTables is the exact table set Stalwart's own v0.16 upgrade guide
|
||||
// backs up before migrating (principals/directory, domains, and the other
|
||||
// tables its migration script and recovery mode depend on) - a targeted
|
||||
// dump, not a full-instance one, matching the guide's own tested restore
|
||||
// path and staying fast on large installs. See ARCHITECTURE.md §4.2 and
|
||||
// UPGRADING/v0_16.md.
|
||||
var criticalTables = []string{"s", "d", "r", "h", "b", "g", "j", "f", "u"}
|
||||
|
||||
// SQLOptions configures a targeted critical-table dump for an external SQL
|
||||
// store backend.
|
||||
type SQLOptions struct {
|
||||
Host string
|
||||
Port string
|
||||
Database string
|
||||
User string
|
||||
Password string
|
||||
OutPath string
|
||||
}
|
||||
|
||||
// BuildPgDumpArgs returns pg_dump's argv for the critical-table backup,
|
||||
// without the leading "pg_dump" itself.
|
||||
func BuildPgDumpArgs(o SQLOptions) []string {
|
||||
args := []string{"-U", o.User, "-d", o.Database}
|
||||
if o.Host != "" {
|
||||
args = append(args, "-h", o.Host)
|
||||
}
|
||||
if o.Port != "" {
|
||||
args = append(args, "-p", o.Port)
|
||||
}
|
||||
for _, t := range criticalTables {
|
||||
args = append(args, "-t", t)
|
||||
}
|
||||
return append(args, "-f", o.OutPath)
|
||||
}
|
||||
|
||||
// BuildMySQLDumpArgs returns mysqldump's argv for the same critical-table
|
||||
// set. mysqldump writes to stdout, so RunMySQLDump redirects it rather than
|
||||
// this function taking an output path flag.
|
||||
func BuildMySQLDumpArgs(o SQLOptions) []string {
|
||||
args := []string{"-u", o.User, o.Database}
|
||||
if o.Host != "" {
|
||||
args = append(args, "-h", o.Host)
|
||||
}
|
||||
if o.Port != "" {
|
||||
args = append(args, "-P", o.Port)
|
||||
}
|
||||
return append(args, criticalTables...)
|
||||
}
|
||||
|
||||
// RunPgDump executes pg_dump with the password passed via the standard
|
||||
// PGPASSWORD environment variable, never on the command line where it would
|
||||
// be visible to anything reading this process's argv (e.g. `ps`).
|
||||
func RunPgDump(ctx context.Context, o SQLOptions) error {
|
||||
cmd := exec.CommandContext(ctx, "pg_dump", BuildPgDumpArgs(o)...)
|
||||
cmd.Env = append(os.Environ(), "PGPASSWORD="+o.Password)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: pg_dump failed: %w (output: %s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunMySQLDump executes mysqldump with the password passed via the standard
|
||||
// MYSQL_PWD environment variable, redirecting its stdout to o.OutPath.
|
||||
func RunMySQLDump(ctx context.Context, o SQLOptions) error {
|
||||
f, err := os.OpenFile(o.OutPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: create %s: %w", o.OutPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "mysqldump", BuildMySQLDumpArgs(o)...)
|
||||
cmd.Env = append(os.Environ(), "MYSQL_PWD="+o.Password)
|
||||
cmd.Stdout = f
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("backup: mysqldump failed: %w (stderr: %s)", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildPgDumpArgs(t *testing.T) {
|
||||
args := BuildPgDumpArgs(SQLOptions{
|
||||
Host: "db.internal", Port: "5432", Database: "stalwart", User: "stalwart", OutPath: "/backups/out.sql",
|
||||
})
|
||||
joined := strings.Join(args, " ")
|
||||
for _, want := range []string{"-U stalwart", "-d stalwart", "-h db.internal", "-p 5432", "-f /backups/out.sql"} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("BuildPgDumpArgs = %q, missing %q", joined, want)
|
||||
}
|
||||
}
|
||||
for _, table := range criticalTables {
|
||||
if !strings.Contains(joined, "-t "+table) {
|
||||
t.Errorf("BuildPgDumpArgs missing critical table flag for %q: %q", table, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMySQLDumpArgs(t *testing.T) {
|
||||
args := BuildMySQLDumpArgs(SQLOptions{Host: "db.internal", Port: "3306", Database: "stalwart", User: "stalwart"})
|
||||
joined := strings.Join(args, " ")
|
||||
if !strings.Contains(joined, "-u stalwart") || !strings.Contains(joined, "-h db.internal") || !strings.Contains(joined, "-P 3306") {
|
||||
t.Errorf("BuildMySQLDumpArgs = %q, missing expected flags", joined)
|
||||
}
|
||||
if !strings.HasSuffix(joined, strings.Join(criticalTables, " ")) {
|
||||
t.Errorf("BuildMySQLDumpArgs = %q, want it to end with the critical table list", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPgDumpInvokesPgDumpWithArgs(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
withFakeExecutable(t, "pg_dump", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
err := RunPgDump(context.Background(), SQLOptions{
|
||||
User: "stalwart", Database: "stalwart", OutPath: filepath.Join(dir, "out.sql"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunPgDump: %v", err)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
if !strings.Contains(got, "-U stalwart") {
|
||||
t.Errorf("pg_dump was invoked with %q, missing -U stalwart", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPgDumpPropagatesFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
withFakeExecutable(t, "pg_dump", "#!/bin/sh\necho 'connection refused' >&2\nexit 1\n")
|
||||
|
||||
err := RunPgDump(context.Background(), SQLOptions{User: "stalwart", Database: "stalwart", OutPath: filepath.Join(dir, "out.sql")})
|
||||
if err == nil {
|
||||
t.Fatal("RunPgDump should have returned an error when pg_dump exits non-zero")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "connection refused") {
|
||||
t.Errorf("error = %v, want it to include pg_dump's stderr", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMySQLDumpWritesStdoutToOutPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
withFakeExecutable(t, "mysqldump", "#!/bin/sh\necho '-- fake dump output'\n")
|
||||
|
||||
outPath := filepath.Join(dir, "out.sql")
|
||||
err := RunMySQLDump(context.Background(), SQLOptions{User: "stalwart", Database: "stalwart", OutPath: outPath})
|
||||
if err != nil {
|
||||
t.Fatalf("RunMySQLDump: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "fake dump output") {
|
||||
t.Errorf("out.sql = %q, want fake dump output redirected into it", data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// VandelayOptions configures a per-account content export via Stalwart's
|
||||
// own Vandelay import/export tool - the documented, backend-independent
|
||||
// backup mechanism (ARCHITECTURE.md §4.2's belt-and-suspenders layer): each
|
||||
// account's mail, calendars, contacts, Sieve scripts, and identities land in
|
||||
// one self-contained SQLite archive.
|
||||
type VandelayOptions struct {
|
||||
BinaryPath string // defaults to "vandelay"
|
||||
URL string // the source JMAP server
|
||||
AuthBasic string // "user:app-password", per vandelay's own --auth-basic flag
|
||||
OutDir string // one <account>.sqlite file per account goes here
|
||||
}
|
||||
|
||||
// BuildVandelayImportArgs returns vandelay's argv for exporting one
|
||||
// account's content into a self-contained SQLite archive, without the
|
||||
// leading "vandelay" itself. "import" is vandelay's own verb for this - it
|
||||
// names the direction relative to the archive file, not the live server;
|
||||
// see ARCHITECTURE.md §4.2.
|
||||
func BuildVandelayImportArgs(o VandelayOptions, accountName, outFile string) []string {
|
||||
return []string{
|
||||
"import", "jmap",
|
||||
"--url", o.URL,
|
||||
"--auth-basic", o.AuthBasic,
|
||||
"--account-name", accountName,
|
||||
outFile,
|
||||
}
|
||||
}
|
||||
|
||||
// ExportAccount runs one account's Vandelay export and returns the archive
|
||||
// path it wrote.
|
||||
func ExportAccount(ctx context.Context, o VandelayOptions, accountName string) (outFile string, err error) {
|
||||
binary := o.BinaryPath
|
||||
if binary == "" {
|
||||
binary = "vandelay"
|
||||
}
|
||||
outFile = filepath.Join(o.OutDir, accountName+".sqlite")
|
||||
cmd := exec.CommandContext(ctx, binary, BuildVandelayImportArgs(o, accountName, outFile)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup: vandelay export of %s failed: %w (output: %s)", accountName, err, out)
|
||||
}
|
||||
return outFile, nil
|
||||
}
|
||||
|
||||
// ExportAccounts exports every account in accounts, stopping at the first
|
||||
// failure. A partial per-account backup set is reported precisely (how many
|
||||
// succeeded, which one failed and why) rather than silently continuing past
|
||||
// a failure that might indicate a systemic problem - bad credentials, an
|
||||
// unreachable server - rather than a one-off.
|
||||
func ExportAccounts(ctx context.Context, o VandelayOptions, accounts []string) (outFiles []string, err error) {
|
||||
for _, acct := range accounts {
|
||||
f, err := ExportAccount(ctx, o, acct)
|
||||
if err != nil {
|
||||
return outFiles, fmt.Errorf("backup: exported %d/%d accounts before failing: %w", len(outFiles), len(accounts), err)
|
||||
}
|
||||
outFiles = append(outFiles, f)
|
||||
}
|
||||
return outFiles, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildVandelayImportArgs(t *testing.T) {
|
||||
args := BuildVandelayImportArgs(VandelayOptions{URL: "https://mail.example.com", AuthBasic: "alice:app-pass"}, "[email protected]", "/backups/alice.sqlite")
|
||||
joined := strings.Join(args, " ")
|
||||
for _, want := range []string{"import", "jmap", "--url https://mail.example.com", "--auth-basic alice:app-pass", "--account-name [email protected]", "/backups/alice.sqlite"} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("BuildVandelayImportArgs = %q, missing %q", joined, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportAccountsRunsEachAccount(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
log := argsFile(t, dir)
|
||||
withFakeExecutable(t, "vandelay", fakeScriptLoggingArgs(log, "exit 0"))
|
||||
|
||||
outDir := filepath.Join(dir, "out")
|
||||
files, err := ExportAccounts(context.Background(), VandelayOptions{URL: "https://mail.example.com", AuthBasic: "a:b", OutDir: outDir}, []string{"[email protected]", "[email protected]"})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAccounts: %v", err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("got %d files, want 2: %v", len(files), files)
|
||||
}
|
||||
got := readArgsFile(t, log)
|
||||
if !strings.Contains(got, "[email protected]") || !strings.Contains(got, "[email protected]") {
|
||||
t.Errorf("vandelay invocations = %q, want both accounts", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportAccountsStopsAtFirstFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Fails on the second invocation (bob), succeeds on the first (alice).
|
||||
script := "#!/bin/sh\ncase \"$*\" in\n *bob*) echo 'account not found' >&2; exit 1 ;;\n *) exit 0 ;;\nesac\n"
|
||||
withFakeExecutable(t, "vandelay", script)
|
||||
|
||||
outDir := filepath.Join(dir, "out")
|
||||
files, err := ExportAccounts(context.Background(), VandelayOptions{URL: "https://mail.example.com", AuthBasic: "a:b", OutDir: outDir}, []string{"[email protected]", "[email protected]", "[email protected]"})
|
||||
if err == nil {
|
||||
t.Fatal("ExportAccounts should have failed on [email protected]")
|
||||
}
|
||||
if len(files) != 1 {
|
||||
t.Errorf("got %d successful exports before failure, want 1 (alice only)", len(files))
|
||||
}
|
||||
if !strings.Contains(err.Error(), "1/3") {
|
||||
t.Errorf("error = %v, want it to report 1/3 accounts exported before failing", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user