A dry run against a real 0.15.5 instance failed with:
recovery mode did not come up: http://127.0.0.1:8081/ did not become
reachable within 1m0s: connect: connection refused
Stalwart had explained itself immediately - "Failed to bind to [::]:8080:
Address already in use" - into a pipe nothing was reading. Diagnosing a
one-line problem took several rounds because the tool threw away the only
evidence. Anything that reports a supervised process failing has to be able
to say why.
Process now captures the child's combined stdout and stderr into a bounded
buffer (64 KiB, keeping the most recent output, with truncation marked
rather than silent - a dead server's reason is at the end of its log), and
exposes it via Output(). recovery.Run appends it to both the startup-timeout
and settings-apply failures, and validate.BootCheck to its boot failure.
Fixes a second bug found while testing the first: Stop returned early when
Signal reported the process had already exited, so cmd.Wait was never
called. Wait is what reaps the child AND waits for the goroutines copying
its output - so the output was discarded in exactly the case where it
matters most, the server dying on its own. os.ErrProcessDone is now treated
as "already gone, still reap it".
The test reproduces the original failure shape: hold the port, start the
helper, let the health check time out, and assert the child's own bind
error survived. Confirmed against the smoke VM too - the same run now ends
with Stalwart's "Address already in use (os error 98)" printed inside the
tool's error.
125 lines
4.7 KiB
Go
125 lines
4.7 KiB
Go
// SPDX-FileCopyrightText: 2026 LINUXexpert-org
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package recovery
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/LINUXexpert-org/stalwart-migrator/internal/checkpoint"
|
|
)
|
|
|
|
// Options configures one full recovery-mode migration cycle: starting the
|
|
// target binary in recovery mode, waiting for it to come up, applying the
|
|
// settings snapshot(s), and stopping it again. See ARCHITECTURE.md §4.4.
|
|
type Options struct {
|
|
BinaryPath string
|
|
ConfigPath string
|
|
ListenURL string // recovery mode's own HTTP listener, e.g. "http://127.0.0.1:8080"
|
|
AdminUser string
|
|
ApplyFiles []string
|
|
CLIBinaryPath string
|
|
ExtraEnv []string // lets a dry-run point ports/paths at a sandbox without touching production config
|
|
StartupTimeout time.Duration
|
|
StopGrace time.Duration
|
|
HTTPClient *http.Client
|
|
}
|
|
|
|
// GenerateRecoveryPassword returns a fresh random one-time password for
|
|
// STALWART_RECOVERY_ADMIN - never the operator's real admin password, never
|
|
// logged, never reused across runs or persisted to the checkpoint.
|
|
func GenerateRecoveryPassword() (string, error) {
|
|
b := make([]byte, 20)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", fmt.Errorf("recovery: generate password: %w", err)
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
// Run executes one recovery-mode cycle as a single checkpointed step. It is
|
|
// deliberately not decomposed into per-sub-step checkpoints the way
|
|
// preflight and backup are: if this tool's own process crashes mid-cycle,
|
|
// the child Stalwart process it started may or may not still be running
|
|
// independently, and blindly "resuming" by reattaching to a guessed PID or
|
|
// killing an unrelated process on the recovery port would be more dangerous
|
|
// than just retrying cleanly. A retry that hits "address already in use"
|
|
// surfaces the real problem (an orphaned process from the failed attempt)
|
|
// for a human to clear, rather than this tool guessing at cleanup.
|
|
//
|
|
// Whatever happens after Start succeeds, Stop is always attempted on the
|
|
// way out (via a deferred call), so a failure partway through this cycle
|
|
// doesn't leak the child process within a single invocation.
|
|
func Run(ctx context.Context, store *checkpoint.Store, rs *checkpoint.RunState, opts Options) (Report, error) {
|
|
var report Report
|
|
|
|
outcome, err := store.RunStep(rs, checkpoint.PhaseRecovery, "recovery-cycle", func() (out checkpoint.StepOutcome, err error) {
|
|
password, err := GenerateRecoveryPassword()
|
|
if err != nil {
|
|
return checkpoint.StepOutcome{}, err
|
|
}
|
|
|
|
proc := &Process{}
|
|
if startErr := proc.Start(ctx, ProcessOptions{
|
|
BinaryPath: opts.BinaryPath, ConfigPath: opts.ConfigPath,
|
|
RecoveryMode: true, AdminUser: opts.AdminUser, AdminPassword: password,
|
|
ExtraEnv: opts.ExtraEnv,
|
|
}); startErr != nil {
|
|
return checkpoint.StepOutcome{}, startErr
|
|
}
|
|
|
|
stopGrace := opts.StopGrace
|
|
if stopGrace <= 0 {
|
|
stopGrace = 10 * time.Second
|
|
}
|
|
defer func() {
|
|
if stopErr := proc.Stop(stopGrace); stopErr != nil && err == nil {
|
|
err = stopErr
|
|
}
|
|
}()
|
|
|
|
startupTimeout := opts.StartupTimeout
|
|
if startupTimeout <= 0 {
|
|
startupTimeout = 60 * time.Second
|
|
}
|
|
if healthErr := WaitForHealthy(ctx, opts.HTTPClient, opts.ListenURL, startupTimeout); healthErr != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("recovery mode did not come up: %w%s", healthErr, outputSuffix(proc))
|
|
}
|
|
|
|
if applyErr := ApplyAll(ctx, ApplyOptions{
|
|
CLIBinaryPath: opts.CLIBinaryPath, URL: opts.ListenURL, User: opts.AdminUser, Password: password,
|
|
}, opts.ApplyFiles); applyErr != nil {
|
|
return checkpoint.StepOutcome{}, fmt.Errorf("settings apply failed: %w%s", applyErr, outputSuffix(proc))
|
|
}
|
|
|
|
return checkpoint.StepOutcome{
|
|
Detail: fmt.Sprintf("recovery mode came up at %s, applied %d settings file(s), stopped cleanly", opts.ListenURL, len(opts.ApplyFiles)),
|
|
}, nil
|
|
})
|
|
|
|
if err != nil {
|
|
report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusFail, Detail: err.Error()})
|
|
return report, err
|
|
}
|
|
report.Results = append(report.Results, CheckResult{Name: "recovery-cycle", Status: StatusOK, Detail: outcome.Detail})
|
|
return report, nil
|
|
}
|
|
|
|
// outputSuffix renders a supervised process's captured output for
|
|
// appending to an error, or nothing if it produced none. The server's own
|
|
// words are usually the whole diagnosis - a bind conflict, a rejected
|
|
// config value - and without them the caller is left guessing at a
|
|
// timeout.
|
|
func outputSuffix(proc *Process) string {
|
|
out := strings.TrimSpace(proc.Output())
|
|
if out == "" {
|
|
return " (the process produced no output)"
|
|
}
|
|
return fmt.Sprintf("\n--- output from the supervised Stalwart process ---\n%s\n--- end of output ---", out)
|
|
}
|