Scale-out storage: PostgreSQL and MySQL read replicas (ST-5 to ST-15)

A data store with readReplicas becomes a replicated store. Writes,
operator-written SQL and everything outside a read scope go to the
primary. JMAP reads before a request's first write, IMAP LIST, STATUS,
SEARCH, SORT and FETCH, POP3 RETR and TOP, DAV GET, PROPFIND and REPORT,
and blob downloads run in a read scope. Only account data (properties,
indexes, change logs, counters, ACLs, blobs, the search index) is read
from a replica; the registry, in-memory values, the task queue and the
rest stay on the primary.

In a scope, the first read picks a replica round-robin among those up
and under the lag limit, and only if it has every change this node has
written or heard of for the scope's accounts: marks come from write
results, the cluster's state-change broadcasts, a sinceState the client
presents, and, with more than one node, Redis. A write inside the scope
sends the rest of it to the primary. A miss on a replica is looked up on
the primary, and a replica error retries the read there and marks the
replica down.

Each node samples lag every second (WAL positions on PostgreSQL; GTID
sets or Seconds_Behind_Source on MySQL), stops reading from a replica
over 5 s and starts again under 2.5 s, and probes a down replica every
10 s. At startup a replica is left out if it's the primary, isn't
read-only, applies out of commit order, or doesn't show a marker written
to the primary within six tries.

replica_tests (postgres, STORE=PostgreSqlReplicated) runs a primary and a
streaming hot standby in containers: tests 9, 10, 12, 13, 14 and 15 pass.

tests/src/store/mod.rs

@@ -10,6 +10,8 @@ pub mod lookup;
 pub mod ops;
 pub mod query;
 pub mod registry;
+#[cfg(feature = "postgres")]
+pub mod replica; // inbuxa: read replicas
 pub mod scaleout; // inbuxa: scale-out storage
 #[cfg(any(feature = "postgres", feature = "mysql"))]
 pub mod sql_timeout;

tests/src/store/replica.rs

@@ -0,0 +1,329 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Coffey Labs
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+//! Read-replica acceptance tests, from
+//! `docs/spec/features/scale-out-storage.md` (tests 9, 10, 12, 13, 14 and
+//! 15), against a PostgreSQL primary with a streaming hot standby in
+//! containers. Built with `postgres`.
+
+use crate::utils::{
+    containers::{
+        PG_PRIMARY_CONTAINER, PG_REPLICA_CONTAINER, PG_REPLICA_PORT, docker, ensure_postgres, psql,
+    },
+    server::TestServerBuilder,
+};
+use jmap_client::email;
+use registry::schema::structs::{
+    Imap, PostgreSqlSettings, PostgreSqlStore, SecretKeyOptional, SecretKeyValue,
+};
+use std::{
+    sync::{Arc, atomic::Ordering},
+    time::{Duration, Instant},
+};
+use store::{
+    Store,
+    backend::scaleout::replica::{ReplicaState, ReplicatedStore},
+};
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+
+const SECRET: &str = "replica test user passphrase";
+
+async fn wait_for(store: &ReplicatedStore, state: ReplicaState, within: Duration, what: &str) {
+    let deadline = Instant::now() + within;
+    while store.replicas[0].state() != state {
+        assert!(
+            Instant::now() < deadline,
+            "{what}: the replica is {:?}, not {state:?}",
+            store.replicas[0].state()
+        );
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+}
+
+fn reads(store: &ReplicatedStore) -> u64 {
+    store.replicas[0].reads.load(Ordering::Relaxed)
+}
+
+fn message(subject: &str) -> Vec<u8> {
+    format!(
+        "From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\r\nBody of {subject}, with the word zebrafish.\r\n"
+    )
+    .into_bytes()
+}
+
+async fn imap_fetch() -> String {
+    let stream = tokio::net::TcpStream::connect("127.0.0.1:9991")
+        .await
+        .unwrap();
+    let (reader, mut writer) = tokio::io::split(stream);
+    let mut lines = BufReader::new(reader).lines();
+    lines.next_line().await.unwrap();
+    let mut transcript = String::new();
+    for (tag, command) in [
+        ("a", format!("LOGIN \"[email protected]\" \"{SECRET}\"")),
+        ("b", "SELECT INBOX".to_string()),
+        ("c", "FETCH 1 BODY.PEEK[]".to_string()),
+        ("d", "LOGOUT".to_string()),
+    ] {
+        writer
+            .write_all(format!("{tag} {command}\r\n").as_bytes())
+            .await
+            .unwrap();
+        while let Ok(Ok(Some(line))) =
+            tokio::time::timeout(Duration::from_secs(10), lines.next_line()).await
+        {
+            transcript.push_str(&line);
+            transcript.push('\n');
+            if line.starts_with(&format!("{tag} ")) {
+                break;
+            }
+        }
+    }
+    transcript
+}
+
+/// `cargo test -p tests --features postgres replica_tests -- --ignored`,
+/// with `STORE=PostgreSqlReplicated`.
+#[ignore]
+#[tokio::test(flavor = "multi_thread")]
+pub async fn replica_tests() {
+    assert_eq!(
+        std::env::var("STORE").as_deref(),
+        Ok("PostgreSqlReplicated"),
+        "run with STORE=PostgreSqlReplicated"
+    );
+    let test = TestServerBuilder::new("replica_tests")
+        .await
+        .with_default_listeners()
+        .await
+        .with_object(Imap {
+            allow_plain_text_auth: true,
+            ..Default::default()
+        })
+        .await
+        .build()
+        .await;
+    let replicated: Arc<ReplicatedStore> = match test.server.store() {
+        Store::Replicated(store) => store.clone(),
+        other => panic!("test 9: the data store isn't replicated: {other:?}"),
+    };
+    // ST-15: checked and in use
+    wait_for(
+        &replicated,
+        ReplicaState::Up,
+        Duration::from_secs(60),
+        "ST-15",
+    )
+    .await;
+
+    let admin = test.account("admin");
+    let user = admin
+        .create_user_account("[email protected]", SECRET, "Replica", &[], vec![])
+        .await;
+    let client = user.jmap_client().await;
+    let inbox = client
+        .mailbox_query(
+            jmap_client::mailbox::query::Filter::role(jmap_client::mailbox::Role::Inbox).into(),
+            None::<Vec<_>>,
+        )
+        .await
+        .unwrap()
+        .take_ids()
+        .pop()
+        .unwrap();
+
+    // Test 9: reads are served by the replica (ST-6)
+    let first = client
+        .email_import(message("First"), [inbox.clone()], None::<Vec<String>>, None)
+        .await
+        .unwrap()
+        .take_id();
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    let before = reads(&replicated);
+    assert!(
+        client
+            .email_get(&first, None::<Vec<_>>)
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert!(
+        reads(&replicated) > before,
+        "test 9: Email/get on the replica"
+    );
+    let before = reads(&replicated);
+    let transcript = imap_fetch().await;
+    assert!(
+        transcript.contains("Subject: First"),
+        "test 9: {transcript}"
+    );
+    assert!(reads(&replicated) > before, "test 9: FETCH on the replica");
+
+    // Test 15: full-text search through the replicated store (ST-3, ST-6)
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    let found = client
+        .email_query(
+            email::query::Filter::text("zebrafish").into(),
+            None::<Vec<_>>,
+        )
+        .await
+        .unwrap()
+        .take_ids();
+    assert!(found.contains(&first), "test 15: {found:?}");
+
+    // Test 10: with replay paused, a client still reads its own writes (ST-7, ST-8)
+    let state = client
+        .email_changes("n", None)
+        .await
+        .unwrap()
+        .new_state()
+        .to_string();
+    psql(PG_REPLICA_CONTAINER, "SELECT pg_wal_replay_pause()");
+    let paused = client
+        .email_import(
+            message("Paused"),
+            [inbox.clone()],
+            None::<Vec<String>>,
+            None,
+        )
+        .await
+        .unwrap()
+        .take_id();
+    assert!(
+        client
+            .email_get(&paused, None::<Vec<_>>)
+            .await
+            .unwrap()
+            .is_some(),
+        "test 10: a new message reads back"
+    );
+    let changes = client.email_changes(state, None).await.unwrap();
+    assert!(
+        changes.created().contains(&paused),
+        "test 10: Email/changes"
+    );
+    let renamed = client
+        .mailbox_create("Before", None::<String>, jmap_client::mailbox::Role::None)
+        .await
+        .unwrap()
+        .take_id();
+    client.mailbox_rename(&renamed, "After").await.unwrap();
+    assert_eq!(
+        client
+            .mailbox_get(&renamed, None::<Vec<_>>)
+            .await
+            .unwrap()
+            .unwrap()
+            .name(),
+        Some("After"),
+        "test 10: a renamed mailbox"
+    );
+
+    // Test 12: over the lag limit, no reads; back once caught up (ST-10, ST-11)
+    let deadline = Instant::now() + Duration::from_secs(20);
+    while replicated.replicas[0].state() != ReplicaState::Lagging {
+        assert!(Instant::now() < deadline, "test 12: never over the limit");
+        client
+            .email_import(message("Lag"), [inbox.clone()], None::<Vec<String>>, None)
+            .await
+            .unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+    let before = reads(&replicated);
+    assert!(
+        client
+            .email_get(&first, None::<Vec<_>>)
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert_eq!(
+        reads(&replicated),
+        before,
+        "test 12: no reads while lagging"
+    );
+    psql(PG_REPLICA_CONTAINER, "SELECT pg_wal_replay_resume()");
+    wait_for(
+        &replicated,
+        ReplicaState::Up,
+        Duration::from_secs(20),
+        "test 12",
+    )
+    .await;
+
+    // Test 13: the replica stopped; requests still succeed (ST-12)
+    docker("stop", PG_REPLICA_CONTAINER);
+    for _ in 0..3 {
+        assert!(
+            client
+                .email_get(&first, None::<Vec<_>>)
+                .await
+                .unwrap()
+                .is_some(),
+            "test 13: served while the replica is down"
+        );
+    }
+    wait_for(
+        &replicated,
+        ReplicaState::Down,
+        Duration::from_secs(15),
+        "test 13",
+    )
+    .await;
+    docker("start", PG_REPLICA_CONTAINER);
+    wait_for(
+        &replicated,
+        ReplicaState::Up,
+        Duration::from_secs(60),
+        "test 13: back",
+    )
+    .await;
+
+    // Test 14: a replica that is writable and unrelated, or the primary
+    // itself, is left out (ST-15)
+    ensure_postgres().await;
+    let setting = |port: u16| PostgreSqlSettings {
+        host: "localhost".into(),
+        port: port as u64,
+        database: "stalwart".into(),
+        auth_username: "stalwart".to_string().into(),
+        auth_secret: SecretKeyOptional::Value(SecretKeyValue {
+            secret: "stalwart".into(),
+        }),
+        options: None,
+    };
+    for (port, what) in [
+        (5432, "an unrelated database"),
+        (5442, "the primary itself"),
+    ] {
+        let store = store::backend::postgres::PostgresStore::open(PostgreSqlStore {
+            host: "localhost".into(),
+            port: 5442,
+            auth_username: "stalwart".to_string().into(),
+            auth_secret: SecretKeyOptional::Value(SecretKeyValue {
+                secret: "stalwart".into(),
+            }),
+            database: "stalwart".into(),
+            read_replicas: registry::types::list::List::from_iter([setting(port)]),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+        let Store::Replicated(store) = store else {
+            panic!("test 14")
+        };
+        wait_for(
+            &store,
+            ReplicaState::Excluded,
+            Duration::from_secs(20),
+            &format!("test 14: {what}"),
+        )
+        .await;
+    }
+    let _ = (PG_PRIMARY_CONTAINER, PG_REPLICA_PORT);
+
+    test.temp_dir.delete();
+}

tests/src/utils/containers.rs

@@ -32,6 +32,14 @@ static CHALLTESTSRV: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_ne
 static PEBBLE: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
 static POWERDNS: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
 static SCIM_TESTER: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+// inbuxa: scale-out storage (ST-5 to ST-15): a primary and a streaming replica
+static PG_PRIMARY: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+static PG_REPLICA: OnceCell<ContainerAsync<GenericImage>> = OnceCell::const_new();
+const PG_REPLICATION_NETWORK: &str = "inbuxa-test-pg-replication";
+pub const PG_PRIMARY_PORT: u16 = 5442;
+pub const PG_REPLICA_PORT: u16 = 5443;
+pub const PG_REPLICA_CONTAINER: &str = "inbuxa-test-pg-replica";
+pub const PG_PRIMARY_CONTAINER: &str = "inbuxa-test-pg-primary";
 
 const OPENLDAP_LDAPI_URL: &str = "ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi/";
 
@@ -541,3 +549,91 @@ async fn wait_for_http(url: &str) {
         tokio::time::sleep(Duration::from_millis(500)).await;
     }
 }
+
+/// inbuxa: a PostgreSQL primary on port 5442 with a hot-standby streaming
+/// replica on 5443, for the read-replica tests (ST-5 to ST-15).
+pub async fn ensure_postgres_replicated() {
+    PG_PRIMARY
+        .get_or_init(|| async {
+            GenericImage::new("postgres", "16-alpine")
+                .with_wait_for(WaitFor::message_on_stderr(
+                    "database system is ready to accept connections",
+                ))
+                .with_env_var("POSTGRES_USER", "stalwart")
+                .with_env_var("POSTGRES_PASSWORD", "stalwart")
+                .with_env_var("POSTGRES_DB", "stalwart")
+                .with_copy_to(
+                    "/docker-entrypoint-initdb.d/replication.sh",
+                    b"#!/bin/sh\necho 'host replication all all scram-sha-256' >> \"$PGDATA/pg_hba.conf\"\n"
+                        .to_vec(),
+                )
+                .with_cmd([
+                    "postgres",
+                    "-c",
+                    "wal_level=replica",
+                    "-c",
+                    "max_wal_senders=10",
+                    "-c",
+                    "hot_standby=on",
+                ])
+                .with_network(PG_REPLICATION_NETWORK)
+                .with_mapped_port(PG_PRIMARY_PORT, 5432.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name(PG_PRIMARY_CONTAINER)
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start the PostgreSQL primary")
+        })
+        .await;
+    wait_for_tcp(PG_PRIMARY_PORT).await;
+    PG_REPLICA
+        .get_or_init(|| async {
+            GenericImage::new("postgres", "16-alpine")
+                .with_wait_for(WaitFor::message_on_stderr(
+                    "database system is ready to accept read-only connections",
+                ))
+                .with_env_var("PGPASSWORD", "stalwart")
+                .with_cmd([
+                    "sh",
+                    "-c",
+                    concat!(
+                        "if [ ! -s /var/lib/postgresql/replica/PG_VERSION ]; then ",
+                        "until pg_basebackup -h inbuxa-test-pg-primary -U stalwart ",
+                        "-D /var/lib/postgresql/replica -R -X stream; do sleep 1; done; fi; ",
+                        "chown -R postgres:postgres /var/lib/postgresql/replica; ",
+                        "chmod 700 /var/lib/postgresql/replica; ",
+                        "exec su-exec postgres postgres -D /var/lib/postgresql/replica ",
+                        "-c hot_standby=on"
+                    ),
+                ])
+                .with_network(PG_REPLICATION_NETWORK)
+                .with_mapped_port(PG_REPLICA_PORT, 5432.tcp())
+                .with_startup_timeout(READY_TIMEOUT)
+                .with_container_name(PG_REPLICA_CONTAINER)
+                .with_reuse(ReuseDirective::Always)
+                .start()
+                .await
+                .expect("Failed to start the PostgreSQL replica")
+        })
+        .await;
+    wait_for_tcp(PG_REPLICA_PORT).await;
+}
+
+/// inbuxa: runs SQL on a test container, through `psql`.
+pub fn psql(container: &str, sql: &str) -> String {
+    let output = std::process::Command::new("docker")
+        .args(["exec", container, "psql", "-U", "stalwart", "-d", "stalwart", "-tAc", sql])
+        .output()
+        .expect("docker exec");
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+/// inbuxa: stops or starts a test container.
+pub fn docker(action: &str, container: &str) {
+    let status = std::process::Command::new("docker")
+        .args([action, container])
+        .status()
+        .expect("docker");
+    assert!(status.success(), "docker {action} {container}");
+}

tests/src/utils/storage.rs

@@ -62,6 +62,35 @@ impl RegistryEnvStores for RegistryStore {
 }
 
 pub async fn build_data_store(typ: &str, path: &str) -> DataStore {
+    // inbuxa: scale-out storage: a primary with a streaming read replica
+    if typ == "PostgreSqlReplicated" {
+        crate::utils::containers::ensure_postgres_replicated().await;
+        let secret = || {
+            SecretKeyOptional::Value(SecretKeyValue {
+                secret: "stalwart".into(),
+            })
+        };
+        return DataStore::PostgreSql(PostgreSqlStore {
+            host: "localhost".into(),
+            port: crate::utils::containers::PG_PRIMARY_PORT as u64,
+            auth_username: "stalwart".to_string().into(),
+            auth_secret: secret(),
+            database: "stalwart".into(),
+            use_tls: false,
+            allow_invalid_certs: true,
+            read_replicas: registry::types::list::List::from_iter([
+                registry::schema::structs::PostgreSqlSettings {
+                    host: "localhost".into(),
+                    port: crate::utils::containers::PG_REPLICA_PORT as u64,
+                    database: "stalwart".into(),
+                    auth_username: "stalwart".to_string().into(),
+                    auth_secret: secret(),
+                    options: None,
+                },
+            ]),
+            ..Default::default()
+        });
+    }
     if typ == "MariaDb" {
         crate::utils::containers::ensure_mariadb().await;
         return DataStore::MySql(MySqlStore {