diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2857f26..65359ce 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -290,9 +290,28 @@ against an already-migrated store. - The original unit is preserved and recorded as the `service-unit` artifact *before* the rewrite, so an operator restoring by hand isn't reconstructing a unit file from memory. -- Docker deployments are refused: cutting over a container means pulling a - new image and recreating the container, not swapping a binary and - rewriting a unit. +- Docker deployments recreate rather than rewrite, because a container + cannot be edited in place the way a unit file can. That makes silent loss + the default failure: a container rebuilt without its capabilities, its + custom network or its device mappings starts cleanly and is quietly not + the server it was. So the same rule the unit rewrite follows applies + here - a definition this only partly understands is one it must not + rebuild - and cutover refuses a container using anything outside the set + it carries across, naming what it found. The list is conservative and + deliberately not exhaustive; docker's HostConfig has far more fields than + it checks, and one it does not know about is a reason not to be + recreating that container at all. +- The old container is renamed rather than removed, and the old image is + never pruned. Together they are the container's manual restore path, the + nearest equivalent to the preserved binary of §4.2: one command starts + the previous container again. The `docker inspect` of the container as it + was is preserved as the `container-definition` artifact before anything + is replaced, for the same reason the unit file is. +- **Status: the container path is implemented and not yet reachable from + the CLI** - `run` does not pass container options, so a container is + still refused there. Wiring it up, and lifting preflight's refusal for + the containers it can now handle, is the remaining work in + [#3](https://github.com/LINUXexpert-org/stalwart-migrator/issues/3). - Quota recalculation is the one step allowed to fail without failing the phase. Stale counters are an accounting problem; a failed cutover is one an operator has to respond to by restoring a machine that is otherwise @@ -815,9 +834,16 @@ happens to need them. `preflight.DeploymentKind` is a type alias for §4.5 lists exactly which two details are inferred. A smoke test against a real 0.16 instance would settle both, and would let this step be promoted from "warns on failure" to a hard check. -- **Cutover doesn't handle Docker.** It refuses container deployments - outright, since cutting one over means pulling an image and recreating - the container rather than swapping a binary and rewriting a unit. +- **Docker is implemented but not yet wired to the CLI.** Preflight + inspects a container and reports what stands in the way; stage pulls and + verifies an image; the recovery cycle runs in a throwaway container + against the live data; and cutover recreates the container, refusing one + whose definition it would not carry across intact. What is missing is + `run` passing those options and preflight lifting its refusal for the + containers now handled. Compose stays refused deliberately: recreating a + compose-managed container out from under compose leaves the container and + the compose file disagreeing, and the next `compose up` reverts the + migration. - **Cutover ignores systemd drop-ins.** It rewrites only the main unit file, so an `ExecStart` or `Environment` override in `/etc/systemd/system/stalwart.service.d/*.conf` is invisible to it - diff --git a/internal/cutover/container.go b/internal/cutover/container.go new file mode 100644 index 0000000..a95f16a --- /dev/null +++ b/internal/cutover/container.go @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package cutover + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" + "github.com/LINUXexpert-org/stalwart-migrator/internal/preflight" +) + +// ArtifactContainerDefinition is the preserved `docker inspect` of the +// container as it was before cutover replaced it - the container's +// equivalent of ArtifactServiceUnit, and for the same reason. Recovery is +// out of scope (ARCHITECTURE.md §4.8), so what this tool owes an operator +// putting a machine back by hand is the definition they would otherwise be +// reconstructing from memory. +const ArtifactContainerDefinition = "container-definition" + +// ContainerOptions configures cutting over a container deployment. +type ContainerOptions struct { + // ContainerName is the live container, which must already be stopped. + ContainerName string + + // StagedImage is the image ID stage verified - an ID rather than a tag + // deliberately, so what runs is what was checked. + StagedImage string + + // PreserveDir is where the inspected definition is written. + PreserveDir string + + DockerBinary string +} + +func (o ContainerOptions) docker() string { + if o.DockerBinary == "" { + return "docker" + } + return o.DockerBinary +} + +// runContainerCutover replaces the container with one running the staged +// image, carrying across the parts of its definition this tool understands +// and refusing outright when it finds parts it does not. +// +// The old container is renamed rather than removed, and the old image is +// never pruned. Together they are the manual restore path: an operator can +// start the previous container again with one command, which is as close to +// the preserved-binary guarantee (§4.2) as a container gets. +func runContainerCutover(ctx context.Context, rs *checkpoint.RunState, step stepFunc, opts ContainerOptions) error { + if opts.ContainerName == "" { + return fmt.Errorf("cutover: no container name") + } + if opts.StagedImage == "" { + return fmt.Errorf("cutover: no staged image - stage the target image before cutting over to it") + } + + var facts preflight.ContainerFacts + + if err := step("preserve-container-definition", func() (checkpoint.StepOutcome, error) { + raw, err := dockerOut(ctx, opts.docker(), "inspect", opts.ContainerName) + if err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("inspect %s: %w (%s)", opts.ContainerName, err, raw) + } + if opts.PreserveDir == "" { + return checkpoint.StepOutcome{}, fmt.Errorf("no directory to preserve the container definition in") + } + if err := os.MkdirAll(opts.PreserveDir, 0o750); err != nil { + return checkpoint.StepOutcome{}, err + } + dest := filepath.Join(opts.PreserveDir, opts.ContainerName+".inspect.json") + if err := os.WriteFile(dest, []byte(raw), 0o640); err != nil { + return checkpoint.StepOutcome{}, err + } + sum, size, err := hashFile(dest) + if err != nil { + return checkpoint.StepOutcome{}, err + } + // Recorded before anything is replaced, so a crash between + // preserving and recreating still leaves the original findable. + rs.RecordArtifact(ArtifactContainerDefinition, checkpoint.Artifact{Path: dest, SHA256: sum, SizeBytes: size}) + + facts, err = preflight.InspectContainer(ctx, opts.ContainerName) + if err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: "preserved the container definition at " + dest}, nil + }); err != nil { + return err + } + + if err := step("container-is-recreatable", func() (checkpoint.StepOutcome, error) { + if len(facts.Unsupported) > 0 { + return checkpoint.StepOutcome{}, fmt.Errorf( + "this container uses configuration cutting over would not carry across: %s. Recreating it without those would "+ + "start cleanly and quietly not be the server it was, so this tool will not do it. Migrate this one by hand: "+ + "the definition is preserved as %s, and the staged image is %s", + strings.Join(facts.Unsupported, "; "), rs.Artifacts[ArtifactContainerDefinition].Path, opts.StagedImage) + } + return checkpoint.StepOutcome{Detail: "the container's definition is entirely within what a recreate carries across"}, nil + }); err != nil { + return err + } + + retired := opts.ContainerName + "-premigration" + if rs.SourceVersion != "" { + retired += "-" + rs.SourceVersion + } + + if err := step("retire-old-container", func() (checkpoint.StepOutcome, error) { + // Renamed, not removed. The old container plus the old image - which + // nothing here prunes - is what an operator restores by hand. + if out, err := dockerOut(ctx, opts.docker(), "rename", opts.ContainerName, retired); err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("rename %s: %w (%s)", opts.ContainerName, err, out) + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("kept the previous container as %s, still on image %s", retired, facts.Image)}, nil + }); err != nil { + return err + } + + return step("create-container", func() (checkpoint.StepOutcome, error) { + args := []string{"run", "-d", "--name", opts.ContainerName} + if facts.RestartPolicy != "" && facts.RestartPolicy != "no" { + args = append(args, "--restart", facts.RestartPolicy) + } + for _, e := range facts.Env { + // Recovery-mode variables must never survive into a normal + // start: leaving STALWART_RECOVERY_MODE set would recovery-boot + // on every restart, the same footgun §4.5 strips from a unit. + if strings.HasPrefix(e, "STALWART_RECOVERY_") { + continue + } + args = append(args, "-e", e) + } + for _, m := range facts.Mounts { + src := m.Name + if src == "" { + src = m.Source + } + spec := src + ":" + m.Destination + if !m.RW { + spec += ":ro" + } + args = append(args, "-v", spec) + } + for port, bindings := range facts.Ports { + for _, b := range bindings { + spec := b.HostPort + ":" + strings.SplitN(port, "/", 2)[0] + if b.HostIP != "" { + spec = b.HostIP + ":" + spec + } + args = append(args, "-p", spec) + } + } + for k, v := range facts.Labels { + args = append(args, "--label", k+"="+v) + } + args = append(args, opts.StagedImage) + + if out, err := dockerOut(ctx, opts.docker(), args...); err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf( + "create %s from %s: %w (%s). The previous container is still here as %s", + opts.ContainerName, opts.StagedImage, err, out, retired) + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("recreated %s on image %s", opts.ContainerName, opts.StagedImage)}, nil + }) +} + +// stepFunc is cutover.Run's checkpointed step runner, passed in so the +// container path reports through the same Report the systemd path does. +type stepFunc func(name string, fn func() (checkpoint.StepOutcome, error)) error + +func dockerOut(ctx context.Context, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + err := cmd.Run() + return strings.TrimSpace(out.String()), err +} diff --git a/internal/cutover/container_test.go b/internal/cutover/container_test.go new file mode 100644 index 0000000..ca02a46 --- /dev/null +++ b/internal/cutover/container_test.go @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package cutover + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint" +) + +// inspectJSON builds a `docker inspect` document. extra is merged into +// HostConfig so a test can add the configuration a recreate would drop. +func inspectJSON(t *testing.T, extraHost map[string]any, networks map[string]any) string { + t.Helper() + host := map[string]any{ + "PortBindings": map[string]any{"143/tcp": []map[string]string{{"HostIp": "0.0.0.0", "HostPort": "143"}}}, + "RestartPolicy": map[string]any{"Name": "unless-stopped"}, + "NetworkMode": "bridge", + "LogConfig": map[string]any{"Type": "json-file"}, + } + for k, v := range extraHost { + host[k] = v + } + if networks == nil { + networks = map[string]any{"bridge": map[string]any{}} + } + doc := []map[string]any{{ + "Name": "/stalwart", + "Image": "sha256:old", + "Config": map[string]any{ + "Image": "stalwartlabs/stalwart:v0.15.5", + "Env": []string{"TZ=UTC", "STALWART_RECOVERY_MODE=1"}, + }, + "State": map[string]any{"Running": false}, + "Mounts": []map[string]any{{"Type": "volume", "Name": "stalwart-data", "Destination": "/opt/stalwart", "RW": true}}, + "HostConfig": host, + "NetworkSettings": map[string]any{"Networks": networks}, + }} + b, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// fakeDockerCutover installs a docker that records arguments and serves the +// given inspect document. +func fakeDockerCutover(t *testing.T, doc string) (log string) { + t.Helper() + dir := t.TempDir() + log = filepath.Join(dir, "args.log") + inspectFile := filepath.Join(dir, "inspect.json") + if err := os.WriteFile(inspectFile, []byte(doc), 0o644); err != nil { + t.Fatal(err) + } + script := fmt.Sprintf(`#!/bin/sh +echo "$@" >> %q +case "$1" in + inspect) cat %q ;; + rename) exit 0 ;; + run) echo newcontainerid ;; +esac +`, log, inspectFile) + if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return log +} + +func runContainerFor(t *testing.T, doc string) (*checkpoint.RunState, Report, error) { + t.Helper() + fakeDockerCutover(t, doc) + store := checkpoint.NewStore(t.TempDir()) + rs, err := store.Create("0.15.5", "0.16.14") + if err != nil { + t.Fatal(err) + } + var report Report + step := func(name string, fn func() (checkpoint.StepOutcome, error)) error { + outcome, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn) + if err != nil { + report.Results = append(report.Results, CheckResult{Name: name, Status: StatusFail, Detail: err.Error()}) + return err + } + report.Results = append(report.Results, CheckResult{Name: name, Status: StatusOK, Detail: outcome.Detail}) + return nil + } + err = runContainerCutover(context.Background(), rs, step, ContainerOptions{ + ContainerName: "stalwart", StagedImage: "sha256:new", PreserveDir: t.TempDir(), + }) + return rs, report, err +} + +func TestContainerCutoverPreservesTheDefinitionFirst(t *testing.T) { + rs, _, err := runContainerFor(t, inspectJSON(t, nil, nil)) + if err != nil { + t.Fatalf("runContainerCutover: %v", err) + } + art, ok := rs.Artifacts[ArtifactContainerDefinition] + if !ok { + t.Fatal("the container definition was not recorded as an artifact") + } + if art.SHA256 == "" || art.SizeBytes == 0 { + t.Errorf("artifact recorded without a checksum or size: %+v", art) + } + body, err := os.ReadFile(art.Path) + if err != nil { + t.Fatalf("preserved definition unreadable: %v", err) + } + if !strings.Contains(string(body), "stalwart-data") { + t.Error("preserved definition does not contain the container's mounts") + } +} + +// The old container is kept, not removed: with the old image unpruned it is +// the manual restore path (ARCHITECTURE.md §4.8). +func TestContainerCutoverRetiresRatherThanRemoves(t *testing.T) { + log := fakeDockerCutover(t, inspectJSON(t, nil, nil)) + store := checkpoint.NewStore(t.TempDir()) + rs, _ := store.Create("0.15.5", "0.16.14") + step := func(name string, fn func() (checkpoint.StepOutcome, error)) error { + _, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn) + return err + } + if err := runContainerCutover(context.Background(), rs, step, ContainerOptions{ + ContainerName: "stalwart", StagedImage: "sha256:new", PreserveDir: t.TempDir(), + }); err != nil { + t.Fatal(err) + } + args := readLog(t, log) + if !strings.Contains(args, "rename stalwart stalwart-premigration-0.15.5") { + t.Errorf("old container was not retired by rename:\n%s", args) + } + if strings.Contains(args, "rm stalwart") || strings.Contains(args, "image rm") || strings.Contains(args, "prune") { + t.Errorf("cutover removed something it should have kept:\n%s", args) + } +} + +func TestContainerCutoverRecreatesWithTheCarriedSettings(t *testing.T) { + log := fakeDockerCutover(t, inspectJSON(t, nil, nil)) + store := checkpoint.NewStore(t.TempDir()) + rs, _ := store.Create("0.15.5", "0.16.14") + step := func(name string, fn func() (checkpoint.StepOutcome, error)) error { + _, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn) + return err + } + if err := runContainerCutover(context.Background(), rs, step, ContainerOptions{ + ContainerName: "stalwart", StagedImage: "sha256:new", PreserveDir: t.TempDir(), + }); err != nil { + t.Fatal(err) + } + args := readLog(t, log) + for _, want := range []string{ + "run -d --name stalwart", + "--restart unless-stopped", + "-e TZ=UTC", + "-v stalwart-data:/opt/stalwart", + "-p 0.0.0.0:143:143", + "sha256:new", + } { + if !strings.Contains(args, want) { + t.Errorf("recreate missing %q\ngot: %s", want, args) + } + } + // Leaving recovery mode set would recovery-boot on every restart - the + // footgun §4.5 strips from a unit. + if strings.Contains(args, "STALWART_RECOVERY_MODE") { + t.Errorf("recovery-mode env survived into the recreated container:\n%s", args) + } +} + +// The hazard this design exists for: a container carrying configuration a +// recreate would drop must be refused, not quietly rebuilt without it. +func TestContainerCutoverRefusesConfigurationItWouldDrop(t *testing.T) { + for _, tc := range []struct { + name string + host map[string]any + nets map[string]any + wants string + }{ + {"capabilities", map[string]any{"CapAdd": []string{"NET_ADMIN"}}, nil, "capabilities"}, + {"devices", map[string]any{"Devices": []any{map[string]any{}}}, nil, "device mappings"}, + {"sysctls", map[string]any{"Sysctls": map[string]string{"net.core.somaxconn": "1024"}}, nil, "sysctls"}, + {"privileged", map[string]any{"Privileged": true}, nil, "privileged"}, + {"log driver", map[string]any{"LogConfig": map[string]any{"Type": "syslog"}}, nil, "log driver"}, + {"user network", nil, map[string]any{"mailnet": map[string]any{}}, "user-defined network"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, err := runContainerFor(t, inspectJSON(t, tc.host, tc.nets)) + if err == nil { + t.Fatalf("expected a refusal for a container with %s", tc.name) + } + if !strings.Contains(err.Error(), tc.wants) { + t.Errorf("refusal should name %q, got: %v", tc.wants, err) + } + if !strings.Contains(err.Error(), "by hand") { + t.Errorf("refusal should tell the operator what to do instead, got: %v", err) + } + }) + } +} + +// A refusal must happen before anything is touched. +func TestContainerCutoverRefusesBeforeRetiringAnything(t *testing.T) { + log := fakeDockerCutover(t, inspectJSON(t, map[string]any{"Privileged": true}, nil)) + store := checkpoint.NewStore(t.TempDir()) + rs, _ := store.Create("0.15.5", "0.16.14") + step := func(name string, fn func() (checkpoint.StepOutcome, error)) error { + _, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn) + return err + } + if err := runContainerCutover(context.Background(), rs, step, ContainerOptions{ + ContainerName: "stalwart", StagedImage: "sha256:new", PreserveDir: t.TempDir(), + }); err == nil { + t.Fatal("expected a refusal") + } + if args := readLog(t, log); strings.Contains(args, "rename") || strings.Contains(args, "run -d") { + t.Errorf("refusal came after the container was already changed:\n%s", args) + } +} + +func TestContainerCutoverNeedsAStagedImage(t *testing.T) { + store := checkpoint.NewStore(t.TempDir()) + rs, _ := store.Create("0.15.5", "0.16.14") + step := func(name string, fn func() (checkpoint.StepOutcome, error)) error { + _, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn) + return err + } + if err := runContainerCutover(context.Background(), rs, step, ContainerOptions{ + ContainerName: "stalwart", PreserveDir: t.TempDir(), + }); err == nil { + t.Fatal("expected a refusal with no staged image") + } +} + +func readLog(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if os.IsNotExist(err) { + return "" + } + if err != nil { + t.Fatal(err) + } + return string(b) +} diff --git a/internal/cutover/cutover.go b/internal/cutover/cutover.go index a663d39..2d3c4e7 100644 --- a/internal/cutover/cutover.go +++ b/internal/cutover/cutover.go @@ -43,6 +43,12 @@ type Options struct { // moved the old binary aside, so this path is normally empty by now. BinaryPath string + // Container, when set, cuts over a container deployment instead of a + // binary and a unit file. Opt-in: a Docker deployment without it is + // still refused, so a caller that has not been taught to stage an image + // cannot reach a half-built container path by accident. + Container *ContainerOptions + // ServiceUnitPath is the systemd unit to rewrite. ConfigPath, if set, // becomes the unit's --config argument. ServiceUnitPath string @@ -147,9 +153,19 @@ func BuildPlan(rs *checkpoint.RunState, opts Options) (Plan, error) { kind = service.Kind(rs.Topology.DeploymentKind) } if kind == service.Docker { - return p, fmt.Errorf( - "cutover: this run's deployment is a Docker container, where cutting over means pulling a new image and recreating the " + - "container rather than swapping a binary and rewriting a unit. This tool doesn't automate that - do it by hand") + if opts.Container == nil { + return p, fmt.Errorf( + "cutover: this run's deployment is a Docker container, where cutting over means pulling a new image and recreating the " + + "container rather than swapping a binary and rewriting a unit. No container options were given, so there is nothing " + + "to recreate it from - stage the target image and pass them, or do it by hand") + } + if opts.Container.StagedImage == "" { + return p, fmt.Errorf("cutover: no staged image to cut over to - stage the target image first") + } + // The binary checks below are about a file and a unit, neither of + // which a container has. + p.Target = "docker container " + opts.Container.ContainerName + return p, nil } deployment := opts.Deployment deployment.Kind = kind @@ -229,103 +245,118 @@ func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, return nil } - if err := step("verify-staged-binary", func() (checkpoint.StepOutcome, error) { - got, err := preflight.DetectVersion(ctx, plan.StagedBinaryPath) - if err != nil { - return checkpoint.StepOutcome{}, fmt.Errorf("couldn't read the staged binary's version: %w", err) + // A container is replaced rather than edited, so none of the binary + // path applies: there is no file to install and no unit to rewrite. + // What follows either branch - confirming it answers, and the quota + // recalculation - is the same question whatever started it. + if opts.Container != nil { + if err := runContainerCutover(ctx, rs, step, *opts.Container); err != nil { + return report, err } - if rs.TargetVersion != "" && got != rs.TargetVersion { - return checkpoint.StepOutcome{}, fmt.Errorf( - "staged binary %s reports version %s, but this run targets %s - installing it would migrate to a version nobody planned for", - plan.StagedBinaryPath, got, rs.TargetVersion) + if err := step("wait-running", func() (checkpoint.StepOutcome, error) { + if err := service.WaitFor(ctx, controller, true, startTimeoutOr(opts)); err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil + }); err != nil { + return report, err + } + } else { + if err := step("verify-staged-binary", func() (checkpoint.StepOutcome, error) { + got, err := preflight.DetectVersion(ctx, plan.StagedBinaryPath) + if err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("couldn't read the staged binary's version: %w", err) + } + if rs.TargetVersion != "" && got != rs.TargetVersion { + return checkpoint.StepOutcome{}, fmt.Errorf( + "staged binary %s reports version %s, but this run targets %s - installing it would migrate to a version nobody planned for", + plan.StagedBinaryPath, got, rs.TargetVersion) + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("staged binary %s reports %s, matching this run's target", plan.StagedBinaryPath, got), Extra: got}, nil + }); err != nil { + return report, err } - return checkpoint.StepOutcome{Detail: fmt.Sprintf("staged binary %s reports %s, matching this run's target", plan.StagedBinaryPath, got), Extra: got}, nil - }); err != nil { - return report, err - } - if err := step("install-binary", func() (checkpoint.StepOutcome, error) { - sum, size, err := installBinary(plan.StagedBinaryPath, plan.BinaryPath) - if err != nil { - return checkpoint.StepOutcome{}, err + if err := step("install-binary", func() (checkpoint.StepOutcome, error) { + sum, size, err := installBinary(plan.StagedBinaryPath, plan.BinaryPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + rs.RecordArtifact(ArtifactNewBinary, checkpoint.Artifact{Path: plan.BinaryPath, SHA256: sum, SizeBytes: size}) + return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%d bytes)", plan.StagedBinaryPath, plan.BinaryPath, size)}, nil + }); err != nil { + return report, err } - rs.RecordArtifact(ArtifactNewBinary, checkpoint.Artifact{Path: plan.BinaryPath, SHA256: sum, SizeBytes: size}) - return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%d bytes)", plan.StagedBinaryPath, plan.BinaryPath, size)}, nil - }); err != nil { - return report, err - } - if err := step("install-config", func() (checkpoint.StepOutcome, error) { - if plan.ConfigSource == "" { + if err := step("install-config", func() (checkpoint.StepOutcome, error) { + if plan.ConfigSource == "" { + return checkpoint.StepOutcome{ + Verdict: string(StatusSkipped), + Detail: "no converted config to install - the unit is repointed at whatever is already at ConfigPath", + }, nil + } + owner, err := installConfig(plan.ConfigSource, plan.ConfigPath, opts.ConfigOwnerReference) + if err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%s)", plan.ConfigSource, plan.ConfigPath, owner)}, nil + }); err != nil { + return report, err + } + + if err := step("update-service-definition", func() (checkpoint.StepOutcome, error) { + preserved, err := preserveUnit(plan.ServiceUnitPath, rs.RunID) + if err != nil { + return checkpoint.StepOutcome{}, err + } + sum, size, err := hashFile(preserved) + if err != nil { + return checkpoint.StepOutcome{}, err + } + // Recorded before the rewrite, so a crash between preserving and + // rewriting still leaves the original findable. + rs.RecordArtifact(ArtifactServiceUnit, checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size}) + + original, err := os.ReadFile(preserved) + if err != nil { + return checkpoint.StepOutcome{}, fmt.Errorf("cutover: read preserved unit %s: %w", preserved, err) + } + rewritten, err := RewriteUnit(string(original), plan.BinaryPath, plan.ConfigPath) + if err != nil { + return checkpoint.StepOutcome{}, err + } + if err := writeFileAtomic(plan.ServiceUnitPath, []byte(rewritten), 0o644); err != nil { + return checkpoint.StepOutcome{}, err + } return checkpoint.StepOutcome{ - Verdict: string(StatusSkipped), - Detail: "no converted config to install - the unit is repointed at whatever is already at ConfigPath", + Detail: fmt.Sprintf("pointed %s at %s; the original is preserved at %s", plan.ServiceUnitPath, plan.BinaryPath, preserved), + Extra: preserved, }, nil + }); err != nil { + return report, err } - owner, err := installConfig(plan.ConfigSource, plan.ConfigPath, opts.ConfigOwnerReference) - if err != nil { - return checkpoint.StepOutcome{}, err - } - return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%s)", plan.ConfigSource, plan.ConfigPath, owner)}, nil - }); err != nil { - return report, err - } - if err := step("update-service-definition", func() (checkpoint.StepOutcome, error) { - preserved, err := preserveUnit(plan.ServiceUnitPath, rs.RunID) - if err != nil { - return checkpoint.StepOutcome{}, err + if err := step("reload-service-definition", func() (checkpoint.StepOutcome, error) { + if err := controller.ReloadConfig(ctx); err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: "service manager re-read the updated definition"}, nil + }); err != nil { + return report, err } - sum, size, err := hashFile(preserved) - if err != nil { - return checkpoint.StepOutcome{}, err - } - // Recorded before the rewrite, so a crash between preserving and - // rewriting still leaves the original findable. - rs.RecordArtifact(ArtifactServiceUnit, checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size}) - original, err := os.ReadFile(preserved) - if err != nil { - return checkpoint.StepOutcome{}, fmt.Errorf("cutover: read preserved unit %s: %w", preserved, err) + startTimeout := startTimeoutOr(opts) + if err := step("start-service", func() (checkpoint.StepOutcome, error) { + if err := controller.Start(ctx); err != nil { + return checkpoint.StepOutcome{}, err + } + if err := service.WaitFor(ctx, controller, true, startTimeout); err != nil { + return checkpoint.StepOutcome{}, err + } + return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil + }); err != nil { + return report, err } - rewritten, err := RewriteUnit(string(original), plan.BinaryPath, plan.ConfigPath) - if err != nil { - return checkpoint.StepOutcome{}, err - } - if err := writeFileAtomic(plan.ServiceUnitPath, []byte(rewritten), 0o644); err != nil { - return checkpoint.StepOutcome{}, err - } - return checkpoint.StepOutcome{ - Detail: fmt.Sprintf("pointed %s at %s; the original is preserved at %s", plan.ServiceUnitPath, plan.BinaryPath, preserved), - Extra: preserved, - }, nil - }); err != nil { - return report, err - } - - if err := step("reload-service-definition", func() (checkpoint.StepOutcome, error) { - if err := controller.ReloadConfig(ctx); err != nil { - return checkpoint.StepOutcome{}, err - } - return checkpoint.StepOutcome{Detail: "service manager re-read the updated definition"}, nil - }); err != nil { - return report, err - } - - startTimeout := opts.StartTimeout - if startTimeout <= 0 { - startTimeout = 60 * time.Second - } - if err := step("start-service", func() (checkpoint.StepOutcome, error) { - if err := controller.Start(ctx); err != nil { - return checkpoint.StepOutcome{}, err - } - if err := service.WaitFor(ctx, controller, true, startTimeout); err != nil { - return checkpoint.StepOutcome{}, err - } - return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil - }); err != nil { - return report, err } healthTimeout := opts.HealthTimeout @@ -587,3 +618,12 @@ func installConfig(src, dst, reference string) (ownership string, err error) { } return fmt.Sprintf("mode %v, ownership unchanged (no reference file to copy it from)", perm), nil } + +// startTimeoutOr is the wait for the service to come up, defaulted. Shared +// because both branches wait for the same thing. +func startTimeoutOr(opts Options) time.Duration { + if opts.StartTimeout > 0 { + return opts.StartTimeout + } + return 60 * time.Second +} diff --git a/internal/preflight/container.go b/internal/preflight/container.go index 2d4a2e4..9ad27ca 100644 --- a/internal/preflight/container.go +++ b/internal/preflight/container.go @@ -45,6 +45,23 @@ type ContainerFacts struct { Labels map[string]string Mounts []Mount Running bool + + // The rest is what cutover would have to carry across when it recreates + // the container. Recreating is the container equivalent of rewriting a + // unit file, except that a unit can be edited in place and a container + // cannot - so anything not carried here is silently dropped, which is + // the failure UnsupportedForRecreate exists to prevent. + Env []string + Ports map[string][]PortBinding + RestartPolicy string + NetworkMode string + Unsupported []string // populated by UnsupportedForRecreate +} + +// PortBinding is one published port. +type PortBinding struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` } // ComposeProject returns the compose project managing this container, or @@ -95,11 +112,37 @@ type inspectOutput struct { Config struct { Image string `json:"Image"` Labels map[string]string `json:"Labels"` + Env []string `json:"Env"` + User string `json:"User"` } `json:"Config"` State struct { Running bool `json:"Running"` } `json:"State"` - Mounts []Mount `json:"Mounts"` + Mounts []Mount `json:"Mounts"` + HostConfig struct { + PortBindings map[string][]PortBinding `json:"PortBindings"` + RestartPolicy struct { + Name string `json:"Name"` + } `json:"RestartPolicy"` + NetworkMode string `json:"NetworkMode"` + CapAdd []string `json:"CapAdd"` + CapDrop []string `json:"CapDrop"` + Devices []any `json:"Devices"` + Sysctls map[string]string `json:"Sysctls"` + Ulimits []any `json:"Ulimits"` + Privileged bool `json:"Privileged"` + ExtraHosts []string `json:"ExtraHosts"` + DNS []string `json:"Dns"` + GroupAdd []string `json:"GroupAdd"` + SecurityOpt []string `json:"SecurityOpt"` + Tmpfs map[string]string `json:"Tmpfs"` + LogConfig struct { + Type string `json:"Type"` + } `json:"LogConfig"` + } `json:"HostConfig"` + NetworkSettings struct { + Networks map[string]any `json:"Networks"` + } `json:"NetworkSettings"` } // InspectContainer reads the facts about containerName. An error here is @@ -123,14 +166,70 @@ func InspectContainer(ctx context.Context, containerName string) (ContainerFacts return ContainerFacts{}, fmt.Errorf("preflight: docker inspect %s returned no container", containerName) } c := got[0] - return ContainerFacts{ - Name: strings.TrimPrefix(c.Name, "/"), - Image: c.Config.Image, - ImageID: c.Image, - Labels: c.Config.Labels, - Mounts: c.Mounts, - Running: c.State.Running, - }, nil + f := ContainerFacts{ + Name: strings.TrimPrefix(c.Name, "/"), + Image: c.Config.Image, + ImageID: c.Image, + Labels: c.Config.Labels, + Mounts: c.Mounts, + Running: c.State.Running, + Env: c.Config.Env, + Ports: c.HostConfig.PortBindings, + RestartPolicy: c.HostConfig.RestartPolicy.Name, + NetworkMode: c.HostConfig.NetworkMode, + } + f.Unsupported = unsupportedForRecreate(c) + return f, nil +} + +// unsupportedForRecreate names every piece of this container's +// configuration that recreating it would not carry across. +// +// Cutover recreates rather than edits, because a container cannot be edited +// in place the way a unit file can. That makes silent loss the default +// failure: a container recreated without its capabilities, its custom +// network or its device mappings starts cleanly and is quietly not the +// server it was. §4.5 already refuses to edit a unit line it only partly +// understands; this is the same rule, applied where the whole definition +// has to be rebuilt. +// +// The list is deliberately conservative and deliberately not exhaustive - +// docker's HostConfig has far more fields than these. It names the ones a +// mail server plausibly uses, and anything it does not know about is a +// reason this tool should not be recreating that container at all. +func unsupportedForRecreate(c inspectOutput) []string { + var out []string + add := func(cond bool, what string) { + if cond { + out = append(out, what) + } + } + h := c.HostConfig + add(len(h.CapAdd) > 0, "added capabilities (--cap-add)") + add(len(h.CapDrop) > 0, "dropped capabilities (--cap-drop)") + add(len(h.Devices) > 0, "device mappings (--device)") + add(len(h.Sysctls) > 0, "sysctls (--sysctl)") + add(len(h.Ulimits) > 0, "ulimits (--ulimit)") + add(h.Privileged, "privileged mode (--privileged)") + add(len(h.ExtraHosts) > 0, "extra hosts (--add-host)") + add(len(h.DNS) > 0, "custom DNS (--dns)") + add(len(h.GroupAdd) > 0, "supplementary groups (--group-add)") + add(len(h.SecurityOpt) > 0, "security options (--security-opt)") + add(len(h.Tmpfs) > 0, "tmpfs mounts (--tmpfs)") + add(h.LogConfig.Type != "" && h.LogConfig.Type != "json-file", "a non-default log driver (--log-driver "+h.LogConfig.Type+")") + add(c.Config.User != "", "a container user (--user "+c.Config.User+")") + + // A user-defined network is a name in NetworkSettings.Networks that is + // not one of docker's built-ins. Recreating without it puts the server + // somewhere nothing else can reach it. + for name := range c.NetworkSettings.Networks { + switch name { + case "bridge", "host", "none": + default: + out = append(out, "a user-defined network ("+name+")") + } + } + return out } // runContainerChecks adds the checks that only apply to a container. They