The repositories moved off LINUXexpert-org. Here that is not a documentation change: the old organisation was the module path, so it is declared in go.mod and repeated in every internal import. Leaving it would have been worse than a stale link. GitHub redirects the repository, but a go.mod whose module line disagrees with the path it was fetched from is an error rather than a redirect, so `go get` on the new address would have failed against the old declaration. go.mod, 34 files of imports, and the repository links in README and ARCHITECTURE. go mod tidy leaves go.sum untouched -- no dependency moved, only our own path.
630 lines
25 KiB
Go
630 lines
25 KiB
Go
// SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package cutover
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/Coffey-Labs/stalwart-migrator/internal/checkpoint"
|
|
"github.com/Coffey-Labs/stalwart-migrator/internal/preflight"
|
|
"github.com/Coffey-Labs/stalwart-migrator/internal/service"
|
|
"github.com/Coffey-Labs/stalwart-migrator/internal/stalwartapi"
|
|
)
|
|
|
|
// Artifact names this phase records. ArtifactServiceUnit is the preserved
|
|
// copy of the service definition as it was before this phase rewrote it -
|
|
// recovery is out of scope for this tool (see ARCHITECTURE.md §4.8), but
|
|
// the operator putting a machine back by hand should not also have to
|
|
// reconstruct their unit file from memory.
|
|
const (
|
|
ArtifactNewBinary = "new-binary"
|
|
ArtifactServiceUnit = "service-unit"
|
|
)
|
|
|
|
// Options configures the cutover phase - ARCHITECTURE.md §4.5.
|
|
type Options struct {
|
|
// StagedBinaryPath is the already-downloaded target-version binary.
|
|
// Cutover verifies its version before installing it and never
|
|
// downloads anything itself.
|
|
StagedBinaryPath string
|
|
// BinaryPath is where the staged binary is installed - the path the
|
|
// service definition runs. backup's preserve-binary step has already
|
|
// moved the old binary aside, so this path is normally empty by now.
|
|
BinaryPath string
|
|
|
|
// Container, when set, cuts over a container deployment instead of a
|
|
// binary and a unit file. Opt-in: a Docker deployment without it is
|
|
// still refused, so a caller that has not been taught to stage an image
|
|
// cannot reach a half-built container path by accident.
|
|
Container *ContainerOptions
|
|
|
|
// ServiceUnitPath is the systemd unit to rewrite. ConfigPath, if set,
|
|
// becomes the unit's --config argument.
|
|
ServiceUnitPath string
|
|
ConfigPath string
|
|
// ConfigSource, if set, is installed to ConfigPath before the unit is
|
|
// repointed at it - the converted v0.16 config the migration produced.
|
|
// Its ownership and mode are copied from ConfigOwnerReference (the old
|
|
// config, normally), because the service does not run as root and a
|
|
// root-owned config it cannot read fails the service at startup, not at
|
|
// install time. That is not hypothetical: a full migration crash-looped
|
|
// 28 times on "Failed to read data store settings: Permission denied"
|
|
// for exactly this reason.
|
|
ConfigSource string
|
|
ConfigOwnerReference string
|
|
|
|
// RecoveryPointConfirmed is the operator asserting that a recovery
|
|
// point exists for this machine. This tool does not take one, verify
|
|
// one, or restore from one - see ARCHITECTURE.md §4.8 - so this is an
|
|
// acknowledgement, not a check, and BuildPlan refuses without it. An
|
|
// unverifiable assertion is weaker than a guarantee; making it explicit
|
|
// at least means nobody migrates a production mail server having never
|
|
// been asked the question.
|
|
RecoveryPointConfirmed bool
|
|
|
|
Deployment service.Options
|
|
Controller service.Controller
|
|
|
|
StartTimeout time.Duration // waiting for the service to report running; default 60s
|
|
HealthTimeout time.Duration // waiting for it to answer JMAP; default 120s
|
|
|
|
AdminURL string
|
|
AdminUser string
|
|
AdminPassword string
|
|
HTTPClient *http.Client
|
|
|
|
// RecalculateQuotas schedules the post-migration quota rebuild
|
|
// (ARCHITECTURE.md §4.5's last step). It's needed when crossing the
|
|
// 0.15/0.16 boundary, where Stalwart's own upgrade guide says quotas
|
|
// were reset to zero and have to be rebuilt; a patch bump doesn't
|
|
// touch them. TenantIDs additionally rebuilds tenant-level counters,
|
|
// which only multi-tenant installs have.
|
|
RecalculateQuotas bool
|
|
TenantIDs []string
|
|
QuotaTimeout time.Duration // default 30m; large installs legitimately take a while
|
|
}
|
|
|
|
// Plan is what a cutover would do, resolved before anything is touched.
|
|
type Plan struct {
|
|
RunID string
|
|
TargetVersion string
|
|
Target string // what the service controller acts on
|
|
|
|
StagedBinaryPath string
|
|
BinaryPath string
|
|
ServiceUnitPath string
|
|
ConfigPath string
|
|
ConfigSource string
|
|
RecalculateQuotas bool
|
|
}
|
|
|
|
func (p Plan) String() string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "cutover plan for run %s:\n", p.RunID)
|
|
fmt.Fprintf(&b, " 1. confirm %s really is %s, then install it as %s\n", p.StagedBinaryPath, p.TargetVersion, p.BinaryPath)
|
|
if p.ConfigSource != "" {
|
|
fmt.Fprintf(&b, " 2. install %s as %s, owned so the service user can read it\n", p.ConfigSource, p.ConfigPath)
|
|
}
|
|
fmt.Fprintf(&b, " 3. preserve %s, then point its ExecStart at the new binary and strip any recovery-mode env vars\n", p.ServiceUnitPath)
|
|
fmt.Fprintf(&b, " 3. reload the service definition and start %s\n", p.Target)
|
|
fmt.Fprint(&b, " 4. wait for it to answer an authenticated JMAP session request\n")
|
|
if p.RecalculateQuotas {
|
|
fmt.Fprint(&b, " 5. schedule per-account quota recalculation and wait for the task queue to drain\n")
|
|
} else {
|
|
fmt.Fprint(&b, " 5. skip quota recalculation - not needed for this upgrade path\n")
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// BuildPlan resolves the cutover and refuses up front for anything it
|
|
// shouldn't attempt. The most important refusal is the first one: a cutover
|
|
// is only allowed to proceed if this run could still be rolled back, which
|
|
// is what makes "never commit to a change you can't undo" a property of the
|
|
// code rather than a claim in a design document.
|
|
func BuildPlan(rs *checkpoint.RunState, opts Options) (Plan, error) {
|
|
p := Plan{
|
|
RunID: rs.RunID, TargetVersion: rs.TargetVersion,
|
|
StagedBinaryPath: opts.StagedBinaryPath, BinaryPath: opts.BinaryPath,
|
|
ServiceUnitPath: opts.ServiceUnitPath, ConfigPath: opts.ConfigPath,
|
|
ConfigSource: opts.ConfigSource,
|
|
RecalculateQuotas: opts.RecalculateQuotas,
|
|
}
|
|
|
|
if !opts.RecoveryPointConfirmed {
|
|
return p, fmt.Errorf(
|
|
"cutover: no recovery point has been confirmed for this run. This phase migrates a live mail server in place and this " +
|
|
"tool has no way to undo that - recovery is the operator's own snapshot or backup (ARCHITECTURE.md §4.8). Take one, " +
|
|
"verify you can actually restore from it, and re-run confirming that you have")
|
|
}
|
|
|
|
kind := opts.Deployment.Kind
|
|
if kind == "" {
|
|
kind = service.Kind(rs.Topology.DeploymentKind)
|
|
}
|
|
if kind == service.Docker {
|
|
if opts.Container == nil {
|
|
return p, fmt.Errorf(
|
|
"cutover: this run's deployment is a Docker container, where cutting over means pulling a new image and recreating the " +
|
|
"container rather than swapping a binary and rewriting a unit. No container options were given, so there is nothing " +
|
|
"to recreate it from - stage the target image and pass them, or do it by hand")
|
|
}
|
|
if opts.Container.StagedImage == "" {
|
|
return p, fmt.Errorf("cutover: no staged image to cut over to - stage the target image first")
|
|
}
|
|
// The binary checks below are about a file and a unit, neither of
|
|
// which a container has.
|
|
p.Target = "docker container " + opts.Container.ContainerName
|
|
return p, nil
|
|
}
|
|
deployment := opts.Deployment
|
|
deployment.Kind = kind
|
|
controller := opts.Controller
|
|
if controller == nil {
|
|
var err error
|
|
controller, err = service.New(deployment)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
}
|
|
p.Target = controller.Target()
|
|
|
|
if opts.StagedBinaryPath == "" {
|
|
return p, fmt.Errorf("cutover: no staged target binary was given - cutover installs an already-downloaded binary, it doesn't fetch one")
|
|
}
|
|
if _, err := os.Stat(opts.StagedBinaryPath); err != nil {
|
|
return p, fmt.Errorf("cutover: staged binary %s: %w", opts.StagedBinaryPath, err)
|
|
}
|
|
if opts.BinaryPath == "" {
|
|
return p, fmt.Errorf("cutover: no path to install the new binary to was given")
|
|
}
|
|
if opts.ServiceUnitPath == "" {
|
|
return p, fmt.Errorf("cutover: no service definition path was given - without one this phase can't point the service at the new binary")
|
|
}
|
|
if _, err := os.Stat(opts.ServiceUnitPath); err != nil {
|
|
return p, fmt.Errorf("cutover: service definition %s: %w", opts.ServiceUnitPath, err)
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// Run executes ARCHITECTURE.md §4.5. Every step is checkpointed, and the
|
|
// order is chosen so that the irreversible-looking parts happen only after
|
|
// the reversible checks pass: the staged binary's version is confirmed
|
|
// before it's installed, and the service definition is preserved before
|
|
// it's rewritten.
|
|
//
|
|
// Quota recalculation is the one step allowed to fail without failing the
|
|
// cutover, and that's deliberate. Stale quota counters are an accounting
|
|
// problem, while a failed cutover is one an operator has to respond to by
|
|
// restoring a machine that is otherwise migrated and serving mail
|
|
// correctly. Calling for that over a counter would be the worse outcome, so
|
|
// this step warns loudly and tells the operator how to finish it by hand.
|
|
func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) {
|
|
var report Report
|
|
|
|
plan, err := BuildPlan(rs, opts)
|
|
if err != nil {
|
|
report.Results = append(report.Results, CheckResult{Name: "plan", Status: StatusFail, Detail: err.Error()})
|
|
return report, err
|
|
}
|
|
|
|
controller := opts.Controller
|
|
if controller == nil {
|
|
deployment := opts.Deployment
|
|
if deployment.Kind == "" {
|
|
deployment.Kind = service.Kind(rs.Topology.DeploymentKind)
|
|
}
|
|
controller, err = service.New(deployment)
|
|
if err != nil {
|
|
report.Results = append(report.Results, CheckResult{Name: "plan", Status: StatusFail, Detail: err.Error()})
|
|
return report, err
|
|
}
|
|
}
|
|
|
|
step := func(name string, fn func() (checkpoint.StepOutcome, error)) error {
|
|
outcome, err := store.RunStep(rs, checkpoint.PhaseCutover, name, fn)
|
|
if err != nil {
|
|
report.Results = append(report.Results, CheckResult{Name: name, Status: StatusFail, Detail: err.Error()})
|
|
return err
|
|
}
|
|
status := Status(outcome.Verdict)
|
|
if status == "" {
|
|
status = StatusOK
|
|
}
|
|
report.Results = append(report.Results, CheckResult{Name: name, Status: status, Detail: outcome.Detail})
|
|
return nil
|
|
}
|
|
|
|
// A container is replaced rather than edited, so none of the binary
|
|
// path applies: there is no file to install and no unit to rewrite.
|
|
// What follows either branch - confirming it answers, and the quota
|
|
// recalculation - is the same question whatever started it.
|
|
if opts.Container != nil {
|
|
if err := runContainerCutover(ctx, rs, step, *opts.Container); err != nil {
|
|
return report, err
|
|
}
|
|
if err := step("wait-running", func() (checkpoint.StepOutcome, error) {
|
|
if err := service.WaitFor(ctx, controller, true, startTimeoutOr(opts)); err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
} else {
|
|
if err := step("verify-staged-binary", func() (checkpoint.StepOutcome, error) {
|
|
got, err := preflight.DetectVersion(ctx, plan.StagedBinaryPath)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("couldn't read the staged binary's version: %w", err)
|
|
}
|
|
if rs.TargetVersion != "" && got != rs.TargetVersion {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf(
|
|
"staged binary %s reports version %s, but this run targets %s - installing it would migrate to a version nobody planned for",
|
|
plan.StagedBinaryPath, got, rs.TargetVersion)
|
|
}
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("staged binary %s reports %s, matching this run's target", plan.StagedBinaryPath, got), Extra: got}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
if err := step("install-binary", func() (checkpoint.StepOutcome, error) {
|
|
sum, size, err := installBinary(plan.StagedBinaryPath, plan.BinaryPath)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
rs.RecordArtifact(ArtifactNewBinary, checkpoint.Artifact{Path: plan.BinaryPath, SHA256: sum, SizeBytes: size})
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%d bytes)", plan.StagedBinaryPath, plan.BinaryPath, size)}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
if err := step("install-config", func() (checkpoint.StepOutcome, error) {
|
|
if plan.ConfigSource == "" {
|
|
return checkpoint.StepOutcome{
|
|
Verdict: string(StatusSkipped),
|
|
Detail: "no converted config to install - the unit is repointed at whatever is already at ConfigPath",
|
|
}, nil
|
|
}
|
|
owner, err := installConfig(plan.ConfigSource, plan.ConfigPath, opts.ConfigOwnerReference)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("installed %s as %s (%s)", plan.ConfigSource, plan.ConfigPath, owner)}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
if err := step("update-service-definition", func() (checkpoint.StepOutcome, error) {
|
|
preserved, err := preserveUnit(plan.ServiceUnitPath, rs.RunID)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
sum, size, err := hashFile(preserved)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
// Recorded before the rewrite, so a crash between preserving and
|
|
// rewriting still leaves the original findable.
|
|
rs.RecordArtifact(ArtifactServiceUnit, checkpoint.Artifact{Path: preserved, SHA256: sum, SizeBytes: size})
|
|
|
|
original, err := os.ReadFile(preserved)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("cutover: read preserved unit %s: %w", preserved, err)
|
|
}
|
|
rewritten, err := RewriteUnit(string(original), plan.BinaryPath, plan.ConfigPath)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
if err := writeFileAtomic(plan.ServiceUnitPath, []byte(rewritten), 0o644); err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
return checkpoint.StepOutcome{
|
|
Detail: fmt.Sprintf("pointed %s at %s; the original is preserved at %s", plan.ServiceUnitPath, plan.BinaryPath, preserved),
|
|
Extra: preserved,
|
|
}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
if err := step("reload-service-definition", func() (checkpoint.StepOutcome, error) {
|
|
if err := controller.ReloadConfig(ctx); err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
return checkpoint.StepOutcome{Detail: "service manager re-read the updated definition"}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
startTimeout := startTimeoutOr(opts)
|
|
if err := step("start-service", func() (checkpoint.StepOutcome, error) {
|
|
if err := controller.Start(ctx); err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
if err := service.WaitFor(ctx, controller, true, startTimeout); err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("%s is running the migrated instance", controller.Target())}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
}
|
|
|
|
healthTimeout := opts.HealthTimeout
|
|
if healthTimeout <= 0 {
|
|
healthTimeout = 120 * time.Second
|
|
}
|
|
if err := step("wait-healthy", func() (checkpoint.StepOutcome, error) {
|
|
if opts.AdminURL == "" {
|
|
return checkpoint.StepOutcome{
|
|
Verdict: string(StatusSkipped),
|
|
Detail: "no admin URL configured - the service was started, but nothing confirmed it actually answers",
|
|
}, nil
|
|
}
|
|
client := newClient(opts)
|
|
// Liveness first, and on its own. Any response - 401 included -
|
|
// proves the service is up and routing.
|
|
if err := client.WaitForResponse(ctx, healthTimeout); err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("the migrated service started but never answered at %s within %s: %w", opts.AdminURL, healthTimeout, err)
|
|
}
|
|
// Credentials are a separate question, and failing them is not a
|
|
// failed cutover. A config fallback-admin does not survive into
|
|
// v0.16 - its config is just a store pointer, so the old
|
|
// [authentication.fallback-admin] block is gone - so the
|
|
// credentials that worked before the migration routinely stop
|
|
// working after it, on an instance that is otherwise fine.
|
|
if err := client.Ping(ctx); err != nil {
|
|
return checkpoint.StepOutcome{
|
|
Verdict: string(StatusWarn),
|
|
Detail: fmt.Sprintf("migrated instance is up and answering at %s, but these admin credentials no longer work: %v. "+
|
|
"If they were a config fallback-admin, that does not survive the migration - v0.16 keeps its config in the store. "+
|
|
"Authenticate as an account that exists in the directory instead", opts.AdminURL, err),
|
|
}, nil
|
|
}
|
|
return checkpoint.StepOutcome{Detail: fmt.Sprintf("migrated instance answered an authenticated JMAP session request at %s", opts.AdminURL)}, nil
|
|
}); err != nil {
|
|
return report, err
|
|
}
|
|
|
|
// From here on, failures are warnings: the service is up and serving
|
|
// mail, and rolling that back over a quota counter would be worse than
|
|
// leaving the counter stale.
|
|
quotaOutcome, quotaErr := store.RunStep(rs, checkpoint.PhaseCutover, "recalculate-quotas", func() (checkpoint.StepOutcome, error) {
|
|
return recalculateQuotas(ctx, opts)
|
|
})
|
|
switch {
|
|
case quotaErr != nil:
|
|
report.Results = append(report.Results, CheckResult{
|
|
Name: "recalculate-quotas", Status: StatusWarn,
|
|
Detail: fmt.Sprintf("%v - the migration is complete and serving mail; finish this from the WebUI's Tasks panel "+
|
|
"(\"Recalculate disk quotas\"), and note that until it runs, per-account usage counters read low", quotaErr),
|
|
})
|
|
default:
|
|
status := Status(quotaOutcome.Verdict)
|
|
if status == "" {
|
|
status = StatusOK
|
|
}
|
|
report.Results = append(report.Results, CheckResult{Name: "recalculate-quotas", Status: status, Detail: quotaOutcome.Detail})
|
|
}
|
|
|
|
return report, nil
|
|
}
|
|
|
|
func newClient(opts Options) *stalwartapi.Client {
|
|
return &stalwartapi.Client{
|
|
BaseURL: opts.AdminURL, Username: opts.AdminUser, Password: opts.AdminPassword, HTTPClient: opts.HTTPClient,
|
|
}
|
|
}
|
|
|
|
func recalculateQuotas(ctx context.Context, opts Options) (checkpoint.StepOutcome, error) {
|
|
if !opts.RecalculateQuotas {
|
|
return checkpoint.StepOutcome{
|
|
Verdict: string(StatusSkipped),
|
|
Detail: "not needed for this upgrade path - quotas are only reset by the 0.15/0.16 schema migration",
|
|
}, nil
|
|
}
|
|
if opts.AdminURL == "" {
|
|
return checkpoint.StepOutcome{
|
|
Verdict: string(StatusSkipped),
|
|
Detail: "no admin URL configured - quota recalculation has to be triggered from the WebUI's Tasks panel by hand",
|
|
}, nil
|
|
}
|
|
|
|
timeout := opts.QuotaTimeout
|
|
if timeout <= 0 {
|
|
timeout = 30 * time.Minute
|
|
}
|
|
client := newClient(opts)
|
|
|
|
accountIDs, err := client.AccountIDs(ctx)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("couldn't enumerate accounts to recalculate: %w", err)
|
|
}
|
|
if len(accountIDs) == 0 {
|
|
return checkpoint.StepOutcome{Verdict: string(StatusSkipped), Detail: "the instance reports no accounts, so there are no quotas to rebuild"}, nil
|
|
}
|
|
|
|
taskIDs, err := client.CreateQuotaRecalculationTasks(ctx, accountIDs)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
failures, err := client.WaitForTasks(ctx, taskIDs, timeout)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
if len(failures) > 0 {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("%d of %d account quota task(s) failed: %v", len(failures), len(taskIDs), failures)
|
|
}
|
|
detail := fmt.Sprintf("rebuilt disk quotas for %d account(s)", len(accountIDs))
|
|
|
|
// Tenant totals aggregate the per-account numbers, so they can only run
|
|
// once every account task above has finished - which is why this is
|
|
// sequenced after the wait rather than scheduled alongside it.
|
|
if len(opts.TenantIDs) > 0 {
|
|
tenantTasks, err := client.CreateTenantQuotaRecalculationTasks(ctx, opts.TenantIDs)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("%s, but scheduling tenant recalculation failed: %w", detail, err)
|
|
}
|
|
tenantFailures, err := client.WaitForTasks(ctx, tenantTasks, timeout)
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("%s, but waiting on tenant recalculation failed: %w", detail, err)
|
|
}
|
|
if len(tenantFailures) > 0 {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("%s, but %d tenant quota task(s) failed: %v", detail, len(tenantFailures), tenantFailures)
|
|
}
|
|
detail += fmt.Sprintf(" and %d tenant(s)", len(opts.TenantIDs))
|
|
}
|
|
return checkpoint.StepOutcome{Detail: detail}, nil
|
|
}
|
|
|
|
// installBinary copies the staged binary into place through a temp file in
|
|
// the same directory, so the service definition never points at a
|
|
// half-written executable. It's idempotent: a retry that finds the right
|
|
// bytes already installed reports them rather than copying again.
|
|
func installBinary(stagedPath, binaryPath string) (sha256Hex string, size int64, err error) {
|
|
stagedSum, stagedSize, err := hashFile(stagedPath)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if existingSum, existingSize, err := hashFile(binaryPath); err == nil && existingSum == stagedSum {
|
|
return existingSum, existingSize, nil
|
|
}
|
|
|
|
data, err := os.ReadFile(stagedPath)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("cutover: read staged binary %s: %w", stagedPath, err)
|
|
}
|
|
if err := writeFileAtomic(binaryPath, data, 0o755); err != nil {
|
|
return "", 0, err
|
|
}
|
|
return stagedSum, stagedSize, nil
|
|
}
|
|
|
|
// preserveUnit copies the current service definition to
|
|
// "<path>.pre-<run-id>" so an operator restoring by hand has the original,
|
|
// and returns that path.
|
|
// Idempotent: a retry finds the copy already there and keeps it, since the
|
|
// file at path may by then be this phase's own rewrite.
|
|
func preserveUnit(unitPath, runID string) (preservedPath string, err error) {
|
|
preservedPath = fmt.Sprintf("%s.pre-%s", unitPath, runID)
|
|
if _, err := os.Stat(preservedPath); err == nil {
|
|
return preservedPath, nil
|
|
} else if !os.IsNotExist(err) {
|
|
return "", fmt.Errorf("cutover: stat %s: %w", preservedPath, err)
|
|
}
|
|
data, err := os.ReadFile(unitPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cutover: read service definition %s: %w", unitPath, err)
|
|
}
|
|
perm := os.FileMode(0o644)
|
|
if info, err := os.Stat(unitPath); err == nil {
|
|
perm = info.Mode().Perm()
|
|
}
|
|
if err := writeFileAtomic(preservedPath, data, perm); err != nil {
|
|
return "", err
|
|
}
|
|
return preservedPath, nil
|
|
}
|
|
|
|
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
|
tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".tmp-*")
|
|
if err != nil {
|
|
return fmt.Errorf("cutover: create temp file next to %s: %w", path, err)
|
|
}
|
|
tmpPath := tmp.Name()
|
|
defer os.Remove(tmpPath) // no-op once the rename succeeds
|
|
|
|
if _, err := tmp.Write(data); err != nil {
|
|
tmp.Close()
|
|
return fmt.Errorf("cutover: write %s: %w", tmpPath, err)
|
|
}
|
|
if err := tmp.Chmod(perm); err != nil {
|
|
tmp.Close()
|
|
return fmt.Errorf("cutover: chmod %s: %w", tmpPath, err)
|
|
}
|
|
if err := tmp.Sync(); err != nil {
|
|
tmp.Close()
|
|
return fmt.Errorf("cutover: sync %s: %w", tmpPath, err)
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return fmt.Errorf("cutover: close %s: %w", tmpPath, err)
|
|
}
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
return fmt.Errorf("cutover: move %s into place at %s: %w", tmpPath, path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func hashFile(path string) (sha256Hex string, size int64, err error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("cutover: hash %s: %w", path, err)
|
|
}
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, f)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("cutover: hash %s: %w", path, err)
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), n, nil
|
|
}
|
|
|
|
// installConfig places the converted config at dst, copying ownership and
|
|
// mode from reference (normally the config being replaced) so the service
|
|
// user can still read it.
|
|
//
|
|
// Ownership is the whole point of this function. Writing a config as root
|
|
// is the natural thing for a tool running as root to do, and it produces a
|
|
// service that starts, fails to read its own config, and restarts forever -
|
|
// a failure that shows up minutes later in the journal rather than at the
|
|
// moment of the mistake. Where no reference is available the file is left
|
|
// world-readable, since a config the service cannot read is worse than one
|
|
// other local users can.
|
|
func installConfig(src, dst, reference string) (ownership string, err error) {
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cutover: read converted config %s: %w", src, err)
|
|
}
|
|
|
|
perm := os.FileMode(0o644)
|
|
uid, gid := -1, -1
|
|
if reference == "" {
|
|
reference = dst // fall back to whatever is already in place
|
|
}
|
|
if info, statErr := os.Stat(reference); statErr == nil {
|
|
perm = info.Mode().Perm()
|
|
if sys, ok := info.Sys().(*syscall.Stat_t); ok {
|
|
uid, gid = int(sys.Uid), int(sys.Gid)
|
|
}
|
|
}
|
|
|
|
if err := writeFileAtomic(dst, data, perm); err != nil {
|
|
return "", err
|
|
}
|
|
if uid >= 0 && gid >= 0 {
|
|
if err := os.Chown(dst, uid, gid); err != nil {
|
|
return "", fmt.Errorf("cutover: set ownership on %s to %d:%d - the service runs as that user and cannot read a config it does not own: %w", dst, uid, gid, err)
|
|
}
|
|
return fmt.Sprintf("uid %d, gid %d, mode %v", uid, gid, perm), nil
|
|
}
|
|
return fmt.Sprintf("mode %v, ownership unchanged (no reference file to copy it from)", perm), nil
|
|
}
|
|
|
|
// startTimeoutOr is the wait for the service to come up, defaulted. Shared
|
|
// because both branches wait for the same thing.
|
|
func startTimeoutOr(opts Options) time.Duration {
|
|
if opts.StartTimeout > 0 {
|
|
return opts.StartTimeout
|
|
}
|
|
return 60 * time.Second
|
|
}
|