Cutover: rehearsed, and the Enterprise build can't read the fork's store
The runbook said nothing in it had been rehearsed. Now the sequence has been, on data made up for the purpose: upstream 0.16.22 in a container as the install running today, the fork beside it, both unprivileged with CAP_NET_BIND_SERVICE. It rehearses the sequence, not the data, which is what the compat tests are for. 27 of 27 checks passed and the rollback took 1.5 seconds. The question §"Open" asked about carrying a store back is answered, and the answer is no. Upstream refuses to start on a store the fork has opened: "Column families not opened: _". The fork adds one RocksDB column family for masked email (SUBSPACE_INBUXA = b'_') and opens with create_missing_column_families, so it creates it on first open; upstream has no descriptor for it and RocksDB will not open a database holding one it was not told about. That makes step 3's "a copy, not a move" load-bearing in a way the step did not say. One open by the fork is enough: pointing it at the original even once, to check something, leaves the Enterprise install unable to start, and there is no rollback after that. It fails loudly and before reading anything, which is the good version of this failure, but it is not recoverable. Two things the rehearsal found that would have wasted time on the day: memberTenantId does not come down from the domain and is refused on create, so a tenant "admin" set up the obvious way is a server administrator and the check passes while proving nothing; and IMAP's INBOX is not JMAP's account, because mail from an unauthenticated sender is filed as spam, so the two counts differ before and after alike. What the rehearsal does not cover is in its README and in §"Open": systemd and `systemctl disable stalwart` above all, ACME renewal, load, the front ends, and INBUXA's own data.
This commit is contained in:
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
"""Phase 3: the rollback, and the question cutover.md leaves open.
|
||||
|
||||
Rollback, as the plan describes it: stop the fork, bring the old install back
|
||||
on its untouched store. What it costs is whatever the fork accepted while it
|
||||
served — this measures that rather than asserting it.
|
||||
|
||||
Then the open question: point the old build at the store the fork has been
|
||||
writing to. cutover.md says this has never been checked and is worth an hour
|
||||
beforehand rather than an argument at 2am.
|
||||
"""
|
||||
import json, os, subprocess, sys, time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import lib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ADMIN_PW = "cutover-rehearsal-pw"
|
||||
PEOPLE = {"alice": "alice-pw-7f21", "bob": "bob-pw-93c4", "carol": "carol-pw-15ab"}
|
||||
|
||||
|
||||
def mailbox(addr, secret, account_id):
|
||||
q = lib.jmap(addr, secret, [["Email/query", {"accountId": account_id}, "0"]])
|
||||
ids = q[0][1].get("ids", [])
|
||||
subs = []
|
||||
if ids:
|
||||
g = lib.jmap(addr, secret, [["Email/get", {"accountId": account_id, "ids": ids,
|
||||
"properties": ["subject"]}, "0"]])
|
||||
subs = sorted(e.get("subject") or "" for e in g[0][1]["list"])
|
||||
return {"count": len(ids), "subjects": subs}
|
||||
|
||||
|
||||
def main():
|
||||
before = json.load(open(f"{HERE}/before.json"))
|
||||
admin = before["admin"]
|
||||
|
||||
print("== phase 3: the rollback\n")
|
||||
|
||||
# What the fork accepted while it served. This is the cost.
|
||||
after_fork = {n: mailbox(f"{n}@{lib.DOMAIN}", s, before["accounts"][n])
|
||||
for n, s in PEOPLE.items()}
|
||||
gained = {n: after_fork[n]["count"] - before["mail"][n]["count"] for n in PEOPLE}
|
||||
print(" messages the fork accepted that the old store never saw:", gained)
|
||||
|
||||
print("\n-- stop the fork, bring the old install back")
|
||||
t0 = time.time()
|
||||
lib.stop("new")
|
||||
if not lib.start("old"):
|
||||
print(lib.logs("old"))
|
||||
sys.exit("the old install did not come back")
|
||||
elapsed = time.time() - t0
|
||||
lib.check(True, f"rolled back in {elapsed:.1f}s", "stop the fork, start the old unit")
|
||||
|
||||
served = lib.one(admin, ADMIN_PW, "x:Account/query", {})[1]["ids"]
|
||||
lib.check(sorted(served) == before["tenantView"]["serverAccounts"],
|
||||
"every account is back, on the untouched store")
|
||||
|
||||
for name, secret in PEOPLE.items():
|
||||
got = mailbox(f"{name}@{lib.DOMAIN}", secret, before["accounts"][name])
|
||||
want = before["mail"][name]
|
||||
lib.check(got["count"] == want["count"] and got["subjects"] == want["subjects"],
|
||||
f"{name}'s mail is exactly the pre-cutover state",
|
||||
f"{got['count']} vs {want['count']}")
|
||||
|
||||
lost = sum(v for v in gained.values() if v > 0)
|
||||
lib.check(True, f"the rollback leaves {lost} message(s) behind in the fork's store",
|
||||
"the documented cost — the two stores diverge from the moment the fork starts")
|
||||
|
||||
print("\n== the open question: can the old build read a store the fork has written?\n")
|
||||
lib.stop("old")
|
||||
time.sleep(2)
|
||||
|
||||
spec = lib.INSTALLS["new"]
|
||||
name = "cutover-reverse"
|
||||
lib.docker("rm", "-f", name, check_rc=False)
|
||||
lib.docker(
|
||||
"run", "-d", "--name", name,
|
||||
"--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"--cap-add", "NET_BIND_SERVICE",
|
||||
"--entrypoint", "/usr/local/bin/stalwart",
|
||||
"-v", f"{lib.ROOT}/target/debug/stalwart:/usr/local/bin/stalwart:ro",
|
||||
"-v", f"{spec['dir']}/etc:/etc/stalwart",
|
||||
"-v", f"{spec['dir']}/data:/var/lib/stalwart",
|
||||
"-p", f"127.0.0.1:{lib.HTTP}:8080",
|
||||
"--env-file", f"{spec['dir']}/env",
|
||||
lib.IMAGE, "--config", "/etc/stalwart/config.json",
|
||||
)
|
||||
up = lib.wait_http(90)
|
||||
log = (lib.docker("logs", "--tail", "120", name, check_rc=False).stdout or "") + \
|
||||
(lib.docker("logs", "--tail", "120", name, check_rc=False).stderr or "")
|
||||
lib.check(up, "the old build boots on a store the fork has written")
|
||||
|
||||
if up:
|
||||
try:
|
||||
ids = sorted(lib.one(admin, ADMIN_PW, "x:Account/query", {})[1]["ids"])
|
||||
lib.check(ids == before["tenantView"]["serverAccounts"],
|
||||
"the old build reads back every account from the fork's store",
|
||||
f"{len(ids)} accounts")
|
||||
except Exception as e:
|
||||
lib.check(False, "the old build reads back every account from the fork's store",
|
||||
str(e)[:120])
|
||||
try:
|
||||
got = mailbox(f"bob@{lib.DOMAIN}", PEOPLE["bob"], before["accounts"]["bob"])
|
||||
lib.check(got["count"] >= before["mail"]["bob"]["count"],
|
||||
"the old build reads mail the fork delivered", str(got["subjects"]))
|
||||
except Exception as e:
|
||||
lib.check(False, "the old build reads mail the fork delivered", str(e)[:120])
|
||||
errs = [l for l in log.splitlines() if "ERROR" in l]
|
||||
lib.check(not errs, "no errors from the old build on the fork's store",
|
||||
errs[0][:160] if errs else "")
|
||||
|
||||
lib.docker("rm", "-f", name, check_rc=False)
|
||||
ok = lib.summary("phase 3")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user