Scale-out storage: PostgreSQL and MySQL read replicas (ST-5 to ST-15)

A data store with readReplicas becomes a replicated store. Writes,
operator-written SQL and everything outside a read scope go to the
primary. JMAP reads before a request's first write, IMAP LIST, STATUS,
SEARCH, SORT and FETCH, POP3 RETR and TOP, DAV GET, PROPFIND and REPORT,
and blob downloads run in a read scope. Only account data (properties,
indexes, change logs, counters, ACLs, blobs, the search index) is read
from a replica; the registry, in-memory values, the task queue and the
rest stay on the primary.

In a scope, the first read picks a replica round-robin among those up
and under the lag limit, and only if it has every change this node has
written or heard of for the scope's accounts: marks come from write
results, the cluster's state-change broadcasts, a sinceState the client
presents, and, with more than one node, Redis. A write inside the scope
sends the rest of it to the primary. A miss on a replica is looked up on
the primary, and a replica error retries the read there and marks the
replica down.

Each node samples lag every second (WAL positions on PostgreSQL; GTID
sets or Seconds_Behind_Source on MySQL), stops reading from a replica
over 5 s and starts again under 2.5 s, and probes a down replica every
10 s. At startup a replica is left out if it's the primary, isn't
read-only, applies out of commit order, or doesn't show a marker written
to the primary within six tries.

replica_tests (postgres, STORE=PostgreSqlReplicated) runs a primary and a
streaming hot standby in containers: tests 9, 10, 12, 13, 14 and 15 pass.
This commit is contained in:
2026-09-19 14:05:59 -07:00
parent a635b490ec
commit 1518c69033
24 changed files with 1722 additions and 45 deletions
+32 -11
View File
@@ -20,6 +20,8 @@ use mysql_async::{
impl MysqlStore {
pub async fn open(config: structs::MySqlStore) -> Result<Store, String> {
// inbuxa: ST-15: where the primary is, to tell a replica from it
let primary_location = (config.host.clone(), config.port as u16, config.database.clone());
let mut opts = OptsBuilder::default()
.ip_or_hostname(config.host)
.user(config.auth_username)
@@ -51,24 +53,43 @@ impl MysqlStore {
PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()),
);
// inbuxa: ST-2: replicas aren't used yet (the scale-out decision), so
// each one is reported rather than silently ignored
// inbuxa: ST-5 to ST-15: each replica inherits the primary's settings
// except where it is and how to sign in
let mut replicas = vec![];
for replica in config.read_replicas {
trc::event!(
Store(trc::StoreEvent::MysqlError),
Details = format!(
"Read replica {}:{} {} isn't used yet: every operation goes to the primary",
replica.host, replica.port, replica.database
),
);
replicas.push(crate::backend::scaleout::replica::Replica::new(
Store::MySQL(Arc::new(MysqlStore {
conn_pool: Pool::new(
opts.clone()
.ip_or_hostname(replica.host.clone())
.user(replica.auth_username)
.pass(replica.auth_secret.secret().await?.map(|v| v.into_owned()))
.db_name(Some(replica.database.clone()))
.tcp_port(replica.port as u16),
),
})),
replica.host,
replica.port as u16,
replica.database,
));
}
let primary = Store::MySQL(Arc::new(MysqlStore {
conn_pool: Pool::new(opts),
}));
Ok(primary)
// ST-1: no replicas, no change
if replicas.is_empty() {
return Ok(primary);
}
Ok(Store::Replicated(
crate::backend::scaleout::replica::ReplicatedStore::new(
primary,
primary_location,
replicas,
crate::backend::scaleout::replica::ReplicaKind::MySql,
),
))
}
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
+1 -1
View File
@@ -26,7 +26,7 @@ pub struct MysqlStore {
}
#[inline(always)]
fn into_error(err: impl Display) -> trc::Error {
pub(crate) fn into_error(err: impl Display) -> trc::Error {
trc::StoreEvent::MysqlError.reason(err)
}
+42 -11
View File
@@ -27,6 +27,8 @@ use utils::tls::rustls_client_config;
impl PostgresStore {
pub async fn open(config: structs::PostgreSqlStore) -> Result<Store, String> {
// inbuxa: ST-15: where the primary is, to tell a replica from it
let primary_location = (config.host.clone(), config.port as u16, config.database.clone());
let mut cfg = Config::new();
cfg.dbname = config.database.into();
cfg.host = config.host.into();
@@ -57,16 +59,35 @@ impl PostgresStore {
.map_err(|e| format!("Failed to create connection pool: {e}"))?;
let ts_configs = discover_ts_configs(&primary_pool).await;
// inbuxa: ST-2: replicas aren't used yet (the scale-out decision), so
// each one is reported rather than silently ignored
// inbuxa: ST-5 to ST-15: each replica inherits the primary's settings
// except where it is and how to sign in
let mut replicas = vec![];
for replica in config.read_replicas {
trc::event!(
Store(trc::StoreEvent::PostgresqlError),
Details = format!(
"Read replica {}:{} {} isn't used yet: every operation goes to the primary",
replica.host, replica.port, replica.database
),
);
let mut cfg = cfg.clone();
cfg.dbname = replica.database.clone().into();
cfg.host = replica.host.clone().into();
cfg.user = replica.auth_username;
cfg.password = replica.auth_secret.secret().await?.map(|v| v.into_owned());
cfg.port = (replica.port as u16).into();
cfg.options = replica.options;
let pool = if config.use_tls {
cfg.create_pool(
Some(Runtime::Tokio1),
MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)?),
)
} else {
cfg.create_pool(Some(Runtime::Tokio1), NoTls)
}
.map_err(|e| format!("Failed to create connection pool: {e}"))?;
replicas.push(crate::backend::scaleout::replica::Replica::new(
Store::PostgreSQL(Arc::new(PostgresStore {
conn_pool: pool,
ts_configs: ts_configs.clone(),
})),
replica.host,
replica.port as u16,
replica.database,
));
}
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
@@ -74,8 +95,18 @@ impl PostgresStore {
ts_configs,
}));
Ok(primary)
// ST-1: no replicas, no change
if replicas.is_empty() {
return Ok(primary);
}
Ok(Store::Replicated(
crate::backend::scaleout::replica::ReplicatedStore::new(
primary,
primary_location,
replicas,
crate::backend::scaleout::replica::ReplicaKind::PostgreSql,
),
))
}
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
+2 -2
View File
@@ -29,7 +29,7 @@ pub struct PostgresStore {
}
#[inline(always)]
fn into_error(err: tokio_postgres::error::Error) -> trc::Error {
pub(crate) fn into_error(err: tokio_postgres::error::Error) -> trc::Error {
let mut local_err = trc::StoreEvent::PostgresqlError.reason(error_chain(&err));
if let Some(db_err) = err.as_db_error() {
local_err = local_err.code(db_err.code().code().to_string());
@@ -71,7 +71,7 @@ pub(crate) fn is_timeout_error(err: &tokio_postgres::Error) -> bool {
}
#[inline(always)]
fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
pub(crate) fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
match err {
deadpool_postgres::PoolError::Backend(err) => into_error(err),
err => trc::StoreEvent::PostgresqlError.reason(error_chain(&err)),
+21
View File
@@ -12,6 +12,27 @@
pub mod blob;
pub mod layout;
pub mod memory;
pub mod replica;
pub mod replica_health;
/// Calls a PostgreSQL or MySQL backend directly. Replicated stores use it
/// instead of going back through `Store`, whose futures would then contain
/// themselves.
#[macro_export]
#[doc(hidden)]
macro_rules! sql_backend {
($store:expr, $backend:ident => $call:expr) => {
match $store {
#[cfg(feature = "postgres")]
$crate::Store::PostgreSQL($backend) => $call,
#[cfg(feature = "mysql")]
$crate::Store::MySQL($backend) => $call,
_ => Err(trc::StoreEvent::NotConfigured
.into_err()
.details("A replicated store's member isn't PostgreSQL or MySQL")),
}
};
}
pub use blob::ShardedBlobStore;
pub use memory::ShardedInMemoryStore;
@@ -0,0 +1,369 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! SQL read replicas (ST-5 to ST-15). The primary answers everything unless
//! a read is made inside a [`replica_read`] scope, opened by the call sites
//! ST-6 names, for account data only. Inside one, the first read picks a
//! replica that is healthy, within the lag limit, and has caught up with
//! every change this node knows of for the scope's accounts (ST-7);
//! otherwise the scope stays on the primary. A miss or an error on a
//! replica is answered from the primary (ST-8, ST-12).
use crate::{
InMemoryStore, SUBSPACE_ACL, SUBSPACE_BLOB_LINK, SUBSPACE_BLOBS, SUBSPACE_COUNTER,
SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, SUBSPACE_SEARCH_INDEX, Store, ValueKey,
write::{AssignedId, AssignedIds, ValueClass},
};
use ahash::AHashMap;
use std::{
future::Future,
sync::{
Arc, Mutex, OnceLock,
atomic::{AtomicU8, AtomicU64, AtomicUsize, Ordering},
},
};
/// A replica more than this far behind gets no reads (ST-11).
pub const LAG_LIMIT_MS: u64 = 5_000;
/// ... and gets them again under this (ST-11).
pub const LAG_READMIT_MS: u64 = 2_500;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum ReplicaState {
/// Not yet checked at startup (ST-15): no reads.
Unvalidated = 0,
Up = 1,
/// Failed a read or a probe (ST-12): no reads until a probe succeeds.
Down = 2,
/// Over the lag limit (ST-11).
Lagging = 3,
/// Failed a startup check (ST-15): never used.
Excluded = 4,
/// Its lag can't be measured (ST-11): never used.
Unmeasurable = 5,
}
impl ReplicaState {
fn from_u8(value: u8) -> Self {
match value {
1 => ReplicaState::Up,
2 => ReplicaState::Down,
3 => ReplicaState::Lagging,
4 => ReplicaState::Excluded,
5 => ReplicaState::Unmeasurable,
_ => ReplicaState::Unvalidated,
}
}
}
pub struct Replica {
pub store: Store,
/// `host:port database`, for logs.
pub label: String,
/// The connection settings, for ST-15's "is it the primary" check.
pub location: (String, u16, String),
state: AtomicU8,
pub lag_ms: AtomicU64,
/// Reads served, for observability and tests.
pub reads: AtomicU64,
/// Startup marker checks that timed out (ST-15).
pub marker_misses: AtomicU8,
}
impl Replica {
pub fn new(store: Store, host: String, port: u16, database: String) -> Self {
Replica {
store,
label: format!("{host}:{port} {database}"),
location: (host, port, database),
state: AtomicU8::new(ReplicaState::Unvalidated as u8),
lag_ms: AtomicU64::new(u64::MAX),
reads: AtomicU64::new(0),
marker_misses: AtomicU8::new(0),
}
}
pub fn state(&self) -> ReplicaState {
ReplicaState::from_u8(self.state.load(Ordering::Relaxed))
}
pub fn set_state(&self, state: ReplicaState) -> ReplicaState {
ReplicaState::from_u8(self.state.swap(state as u8, Ordering::Relaxed))
}
fn usable(&self) -> bool {
self.state() == ReplicaState::Up
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReplicaKind {
PostgreSql,
MySql,
}
pub struct ReplicatedStore {
pub primary: Store,
/// The primary's connection settings, for ST-15.
pub primary_location: (String, u16, String),
pub replicas: Vec<Replica>,
pub kind: ReplicaKind,
next: AtomicUsize,
/// The highest change id this node has written or heard of, per account
/// (ST-7).
marks: Mutex<AHashMap<u32, u64>>,
/// Shared marks, when more than one node runs (ST-7, step 2).
shared: OnceLock<InMemoryStore>,
}
/// Account data a replica may serve. Everything else (the registry,
/// in-memory values, the task queue, telemetry, the fork's own records)
/// is always read from the primary (ST-5).
pub fn is_replica_subspace(subspace: u8) -> bool {
matches!(
subspace,
SUBSPACE_PROPERTY
| SUBSPACE_INDEXES
| SUBSPACE_LOGS
| SUBSPACE_COUNTER
| SUBSPACE_ACL
| SUBSPACE_BLOB_LINK
| SUBSPACE_BLOBS
| SUBSPACE_SEARCH_INDEX
)
}
/// A read scope: the accounts whose data it reads, each with any change id
/// the client presented (ST-7, step 4), and the replica it settled on.
pub struct ReadScope {
accounts: Vec<(u32, u64)>,
choice: tokio::sync::OnceCell<Option<usize>>,
/// Set by any write made inside the scope: from then on it reads from
/// the primary (ST-6: a read in a request that writes).
wrote: std::sync::atomic::AtomicBool,
}
/// A write happened in the current task: a read scope, if any, stops using
/// replicas.
pub fn note_scope_write() {
let _ = READ_SCOPE.try_with(|scope| scope.wrote.store(true, Ordering::Relaxed));
}
tokio::task_local! {
static READ_SCOPE: Arc<ReadScope>;
}
/// Runs `fut` with replica-eligible reads for `accounts`: `(account, the
/// change id the client presented, or 0)` (ST-6).
pub async fn replica_read<F: Future>(
accounts: impl IntoIterator<Item = (u32, u64)>,
fut: F,
) -> F::Output {
READ_SCOPE
.scope(
Arc::new(ReadScope {
accounts: accounts.into_iter().collect(),
choice: tokio::sync::OnceCell::new(),
wrote: std::sync::atomic::AtomicBool::new(false),
}),
fut,
)
.await
}
/// Carries the current read scope, if any, into a future about to be
/// spawned as its own task (ST-6).
pub fn carry<F: Future>(fut: F) -> impl Future<Output = F::Output> {
let scope = READ_SCOPE.try_with(|scope| scope.clone()).ok();
async move {
match scope {
Some(scope) => READ_SCOPE.scope(scope, fut).await,
None => fut.await,
}
}
}
#[allow(dead_code)] // used with a PostgreSQL or MySQL backend
fn change_id_key(account_id: u32) -> ValueKey<ValueClass> {
ValueKey {
account_id,
collection: 0,
document_id: 0,
class: ValueClass::ChangeId,
}
}
impl ReplicatedStore {
pub fn new(
primary: Store,
primary_location: (String, u16, String),
replicas: Vec<Replica>,
kind: ReplicaKind,
) -> Arc<Self> {
let store = Arc::new(ReplicatedStore {
primary,
primary_location,
replicas,
kind,
next: AtomicUsize::new(0),
marks: Mutex::new(AHashMap::new()),
shared: OnceLock::new(),
});
super::replica_health::spawn(Arc::downgrade(&store));
store
}
/// Shares high-water marks through the in-memory store, for a cluster
/// (ST-7, step 2).
pub fn share_marks(&self, memory: InMemoryStore) {
let _ = self.shared.set(memory);
}
#[allow(dead_code)] // used with the redis feature
fn shared_key(account_id: u32) -> Vec<u8> {
let mut key = b"_rm".to_vec();
key.extend_from_slice(&account_id.to_be_bytes());
key
}
/// Records a change this node made or heard of (ST-7, step 1).
pub fn note_change(&self, account_id: u32, change_id: u64) {
let mut marks = self.marks.lock().unwrap();
let mark = marks.entry(account_id).or_default();
if change_id > *mark {
*mark = change_id;
}
}
/// Records the change ids a write produced, locally and, in a cluster,
/// in the shared store before the write returns (ST-7, steps 1 and 2).
pub async fn note_write(&self, ids: &AssignedIds) {
let mut changes = Vec::new();
for id in &ids.ids {
if let AssignedId::ChangeId(change) = id {
self.note_change(change.account_id, change.change_id);
changes.push((change.account_id, change.change_id));
}
}
#[cfg(feature = "redis")]
if let Some(InMemoryStore::Redis(redis)) = self.shared.get() {
for (account_id, change_id) in changes {
let _ = redis
.key_set(
&Self::shared_key(account_id),
change_id.to_string().as_bytes(),
Some(2 * LAG_LIMIT_MS / 1000),
)
.await;
}
}
#[cfg(not(feature = "redis"))]
let _ = changes;
}
async fn mark(&self, account_id: u32) -> u64 {
let local = self
.marks
.lock()
.unwrap()
.get(&account_id)
.copied()
.unwrap_or_default();
#[cfg(feature = "redis")]
let shared = match self.shared.get() {
Some(InMemoryStore::Redis(redis)) => redis
.key_get::<String>(&Self::shared_key(account_id))
.await
.ok()
.flatten()
.and_then(|mark| mark.parse::<u64>().ok())
.unwrap_or_default(),
_ => 0,
};
#[cfg(not(feature = "redis"))]
let shared = 0;
local.max(shared)
}
/// The replica the current scope reads from, if any (ST-7, ST-13).
pub async fn read_target(&self, subspace: u8) -> Option<usize> {
if !is_replica_subspace(subspace) {
return None;
}
let scope = READ_SCOPE.try_with(|scope| scope.clone()).ok()?;
if scope.wrote.load(Ordering::Relaxed) {
return None;
}
*scope.choice.get_or_init(|| self.choose(&scope)).await
}
async fn choose(&self, scope: &ReadScope) -> Option<usize> {
let count = self.replicas.len();
let start = self.next.fetch_add(1, Ordering::Relaxed);
'replicas: for offset in 0..count {
let index = (start + offset) % count;
let replica = &self.replicas[index];
if !replica.usable() {
continue;
}
for (account_id, presented) in &scope.accounts {
let mark = self.mark(*account_id).await.max(*presented);
if mark == 0 {
continue;
}
let seen: trc::Result<i64> = crate::sql_backend!(
&replica.store,
db => db.get_counter(change_id_key(*account_id)).await
);
match seen {
Ok(seen) if seen as u64 >= mark => {}
// Behind for this account: the primary answers (ST-7)
Ok(_) => return None,
Err(err) => {
self.failed(index, err);
continue 'replicas;
}
}
}
return Some(index);
}
None
}
/// A replica read failed: it's down until a probe succeeds (ST-12).
pub fn failed(&self, index: usize, err: trc::Error) {
let replica = &self.replicas[index];
if replica.set_state(ReplicaState::Down) != ReplicaState::Down {
super::replica_health::report(
self.kind,
format!("Read replica {} is down: {err}", replica.label),
);
}
}
pub fn served(&self, index: usize) {
self.replicas[index].reads.fetch_add(1, Ordering::Relaxed);
}
}
impl Store {
/// Records a change heard from another node (ST-7, step 1). Nothing to
/// do without replicas.
pub fn note_change(&self, account_id: u32, change_id: u64) {
if let Store::Replicated(store) = self {
store.note_change(account_id, change_id);
}
}
/// Shares high-water marks through `memory`, when more than one node
/// runs (ST-7, step 2).
pub fn share_marks(&self, memory: &InMemoryStore) {
if let Store::Replicated(store) = self {
store.share_marks(memory.clone());
}
}
}
@@ -0,0 +1,429 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Each node watches its replicas (ST-10 to ST-15): startup checks, then a
//! lag sample every second, and a probe every ten seconds while one is
//! down. Changes are logged with the backend's existing error event
//! (ST-30).
use super::replica::{
LAG_LIMIT_MS, LAG_READMIT_MS, Replica, ReplicaKind, ReplicaState, ReplicatedStore,
};
use crate::{
SUBSPACE_INBUXA, SerializeInfallible, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
use std::{
collections::VecDeque,
sync::Weak,
time::{Duration, Instant},
};
const SAMPLE_EVERY: Duration = Duration::from_secs(1);
const PROBE_EVERY: Duration = Duration::from_secs(10);
/// Startup marker checks, one a probe, before a replica is left out.
const MARKER_TRIES: u8 = 6;
/// Primary positions kept, one a second.
const SAMPLES_KEPT: usize = 120;
pub fn report(kind: ReplicaKind, message: String) {
match kind {
ReplicaKind::PostgreSql => {
trc::event!(Store(trc::StoreEvent::PostgresqlError), Details = message)
}
ReplicaKind::MySql => trc::event!(Store(trc::StoreEvent::MysqlError), Details = message),
}
}
/// Where the primary is replicated up to, in whatever unit the backend
/// counts (a PostgreSQL LSN, or a MySQL GTID set).
#[allow(dead_code)] // built with a PostgreSQL or MySQL backend
#[derive(Debug, Clone, PartialEq, Eq)]
enum Position {
Lsn(u64),
Gtid(String),
}
pub fn spawn(store: Weak<ReplicatedStore>) {
tokio::spawn(async move {
let mut samples: VecDeque<(Instant, Position)> = VecDeque::new();
let mut last_probe: Vec<Option<Instant>> = Vec::new();
loop {
{
let Some(store) = store.upgrade() else {
return;
};
if last_probe.len() != store.replicas.len() {
last_probe = vec![None; store.replicas.len()];
}
tick(&store, &mut samples, &mut last_probe).await;
}
tokio::time::sleep(SAMPLE_EVERY).await;
}
});
}
async fn tick(
store: &ReplicatedStore,
samples: &mut VecDeque<(Instant, Position)>,
last_probe: &mut [Option<Instant>],
) {
// The primary's position now (ST-10)
match primary_position(store).await {
Ok(Some(position)) => {
if samples.back().is_none_or(|(_, last)| *last != position) {
samples.push_back((Instant::now(), position));
} else if let Some(last) = samples.back_mut() {
// Unchanged: an idle primary; nothing new to wait for
last.0 = last.0.min(Instant::now());
}
while samples.len() > SAMPLES_KEPT {
samples.pop_front();
}
}
Ok(None) => {}
Err(err) => report(
store.kind,
format!("Failed to read the primary's replication position: {err}"),
),
}
for (index, replica) in store.replicas.iter().enumerate() {
let state = replica.state();
match state {
ReplicaState::Excluded | ReplicaState::Unmeasurable => continue,
ReplicaState::Unvalidated | ReplicaState::Down => {
if last_probe[index].is_some_and(|at| at.elapsed() < PROBE_EVERY) {
continue;
}
last_probe[index] = Some(Instant::now());
if state == ReplicaState::Unvalidated {
validate(store, index, replica).await;
continue;
}
}
ReplicaState::Up | ReplicaState::Lagging => {}
}
match replica_lag(store, replica, samples).await {
Ok(Some(lag)) => {
replica
.lag_ms
.store(lag, std::sync::atomic::Ordering::Relaxed);
let next = match state {
ReplicaState::Up if lag > LAG_LIMIT_MS => ReplicaState::Lagging,
ReplicaState::Lagging if lag < LAG_READMIT_MS => ReplicaState::Up,
ReplicaState::Down if lag < LAG_READMIT_MS => ReplicaState::Up,
ReplicaState::Down => ReplicaState::Lagging,
other => other,
};
if next != state {
replica.set_state(next);
report(
store.kind,
format!(
"Read replica {} is {} (lag {lag} ms)",
replica.label,
match next {
ReplicaState::Up => "up",
_ => "over the lag limit",
}
),
);
}
}
Ok(None) => {
if replica.set_state(ReplicaState::Unmeasurable) != ReplicaState::Unmeasurable {
report(
store.kind,
format!(
"Read replica {} gets no reads: its lag can't be measured \
(the REPLICATION CLIENT privilege may be missing)",
replica.label
),
);
}
}
Err(err) if state != ReplicaState::Down => store.failed(index, err),
Err(_) => {}
}
}
}
/// The startup checks (ST-15). A replica that can't be reached yet stays
/// unchecked, and is tried again at the next probe.
async fn validate(store: &ReplicatedStore, index: usize, replica: &Replica) {
let exclude = |why: String| {
replica.set_state(ReplicaState::Excluded);
report(
store.kind,
format!("Read replica {} is left out: {why}", replica.label),
);
};
if replica.location == store.primary_location {
return exclude("it's the primary itself".to_string());
}
match read_only(store.kind, &replica.store).await {
Ok(Ok(())) => {}
Ok(Err(why)) => return exclude(why),
Err(err) => {
report(
store.kind,
format!("Read replica {} can't be checked yet: {err}", replica.label),
);
return;
}
}
// It must be a copy of this primary: a marker written there shows up
let marker = rand::random::<u64>() | 1;
let class = ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: [b'S', b'r', index as u8].to_vec(),
});
let mut batch = BatchBuilder::new();
batch.set(class.clone(), marker.serialize());
if let Err(err) = store.primary.write(batch.build_all()).await {
report(
store.kind,
format!("Failed to write the replica check marker: {err}"),
);
return;
}
let deadline = Instant::now() + Duration::from_millis(LAG_LIMIT_MS);
loop {
match replica
.store
.get_value::<u64>(ValueKey::from(class.clone()))
.await
{
Ok(Some(seen)) if seen == marker => {
replica.set_state(ReplicaState::Up);
report(store.kind, format!("Read replica {} is up", replica.label));
return;
}
Ok(_) if Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(100)).await;
}
// A replica still catching up (after a burst of writes at
// startup, say) gets a few more tries before it's left out
Ok(_) => {
let misses = replica
.marker_misses
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+ 1;
return if misses >= MARKER_TRIES {
exclude("it isn't a copy of this primary".to_string())
} else {
report(
store.kind,
format!(
"Read replica {} hasn't shown the check marker yet; trying again",
replica.label
),
)
};
}
Err(err) => {
report(
store.kind,
format!("Read replica {} can't be checked yet: {err}", replica.label),
);
return;
}
}
}
}
#[allow(unused_variables)]
async fn read_only(kind: ReplicaKind, replica: &Store) -> trc::Result<Result<(), String>> {
match (kind, replica) {
#[cfg(feature = "postgres")]
(ReplicaKind::PostgreSql, Store::PostgreSQL(pg)) => {
let conn = pg
.conn_pool
.get()
.await
.map_err(crate::backend::postgres::into_pool_error)?;
let row = conn
.query_one("SELECT pg_is_in_recovery()", &[])
.await
.map_err(crate::backend::postgres::into_error)?;
Ok(if row.get::<_, bool>(0) {
Ok(())
} else {
Err("it isn't read-only (pg_is_in_recovery() is false)".to_string())
})
}
#[cfg(feature = "mysql")]
(ReplicaKind::MySql, Store::MySQL(my)) => {
use mysql_async::prelude::Queryable;
let mut conn = my
.conn_pool
.get_conn()
.await
.map_err(crate::backend::mysql::into_error)?;
let (read_only, super_read_only): (i64, i64) = conn
.query_first("SELECT @@global.read_only, @@global.super_read_only")
.await
.map_err(crate::backend::mysql::into_error)?
.unwrap_or((0, 0));
if read_only == 0 && super_read_only == 0 {
return Ok(Err(
"it isn't read-only (neither read_only nor super_read_only is on)".to_string(),
));
}
let parallel: Option<(i64, i64)> = conn
.query_first(
"SELECT @@global.replica_parallel_workers, @@global.replica_preserve_commit_order",
)
.await
.ok()
.flatten();
if let Some((workers, preserve)) = parallel
&& workers > 0
&& preserve == 0
{
return Ok(Err(
"it applies in parallel without preserving commit order".to_string(),
));
}
Ok(Ok(()))
}
_ => Ok(Err("the backend isn't compiled in".to_string())),
}
}
#[cfg(feature = "postgres")]
fn parse_lsn(text: &str) -> Option<u64> {
let (high, low) = text.split_once('/')?;
Some((u64::from_str_radix(high, 16).ok()? << 32) | u64::from_str_radix(low, 16).ok()?)
}
#[allow(unused_variables)]
async fn primary_position(store: &ReplicatedStore) -> trc::Result<Option<Position>> {
match &store.primary {
#[cfg(feature = "postgres")]
Store::PostgreSQL(pg) => {
let conn = pg
.conn_pool
.get()
.await
.map_err(crate::backend::postgres::into_pool_error)?;
let row = conn
.query_one("SELECT pg_current_wal_lsn()::text", &[])
.await
.map_err(crate::backend::postgres::into_error)?;
Ok(parse_lsn(row.get::<_, &str>(0)).map(Position::Lsn))
}
#[cfg(feature = "mysql")]
Store::MySQL(my) => {
use mysql_async::prelude::Queryable;
let mut conn = my
.conn_pool
.get_conn()
.await
.map_err(crate::backend::mysql::into_error)?;
let mode: Option<String> = conn
.query_first("SELECT @@global.gtid_mode")
.await
.map_err(crate::backend::mysql::into_error)?;
if mode.as_deref() != Some("ON") {
return Ok(None);
}
let executed: Option<String> = conn
.query_first("SELECT @@global.gtid_executed")
.await
.map_err(crate::backend::mysql::into_error)?;
Ok(executed.map(Position::Gtid))
}
_ => Ok(None),
}
}
/// The replica's lag in milliseconds (ST-10): the age of the oldest
/// primary position it hasn't applied yet. `None` when it can't be
/// measured.
#[allow(unused_variables)]
async fn replica_lag(
store: &ReplicatedStore,
replica: &Replica,
samples: &VecDeque<(Instant, Position)>,
) -> trc::Result<Option<u64>> {
let age = |at: Instant| at.elapsed().as_millis() as u64;
match &replica.store {
#[cfg(feature = "postgres")]
Store::PostgreSQL(pg) => {
let conn = pg
.conn_pool
.get()
.await
.map_err(crate::backend::postgres::into_pool_error)?;
let row = conn
.query_one("SELECT pg_last_wal_replay_lsn()::text", &[])
.await
.map_err(crate::backend::postgres::into_error)?;
let Some(replayed) = row.get::<_, Option<&str>>(0).and_then(parse_lsn) else {
return Err(trc::StoreEvent::PostgresqlError
.into_err()
.details("The replica isn't replaying"));
};
Ok(Some(
samples
.iter()
.find(|(_, position)| matches!(position, Position::Lsn(lsn) if *lsn > replayed))
.map(|(at, _)| age(*at))
.unwrap_or(0),
))
}
#[cfg(feature = "mysql")]
Store::MySQL(my) => {
use mysql_async::prelude::Queryable;
let mut conn = my
.conn_pool
.get_conn()
.await
.map_err(crate::backend::mysql::into_error)?;
if samples.iter().any(|(_, p)| matches!(p, Position::Gtid(_))) {
// With GTIDs: the oldest primary set the replica lacks
for (at, position) in samples {
let Position::Gtid(set) = position else {
continue;
};
let applied: Option<i64> = conn
.exec_first(
"SELECT GTID_SUBSET(?, @@global.gtid_executed)",
(set.as_str(),),
)
.await
.map_err(crate::backend::mysql::into_error)?;
if applied != Some(1) {
return Ok(Some(age(*at)));
}
}
Ok(Some(0))
} else {
// Without: Seconds_Behind_Source, which needs REPLICATION CLIENT
let row: Option<mysql_async::Row> =
match conn.query_first("SHOW REPLICA STATUS").await {
Ok(row) => row,
Err(_) => return Ok(None),
};
let Some(row) = row else {
return Ok(None);
};
match row.get_opt::<Option<u64>, _>("Seconds_Behind_Source") {
Some(Ok(Some(seconds))) => Ok(Some(seconds * 1000)),
Some(Ok(None)) => Err(trc::StoreEvent::MysqlError
.into_err()
.details("Replication is stopped")),
_ => Ok(None),
}
}
}
_ => Ok(None),
}
}
+33
View File
@@ -145,6 +145,33 @@ impl BlobStore {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
// inbuxa: ST-9: a replica, then the primary for what it lacks
Store::Replicated(store) => match store.read_target(crate::SUBSPACE_BLOBS).await {
Some(index) => match crate::sql_backend!(
&store.replicas[index].store,
db => db.get_blob(key, 0..usize::MAX).await
) {
Ok(Some(data)) => {
store.served(index);
Ok(Some(data))
}
Ok(None) => crate::sql_backend!(
&store.primary,
db => db.get_blob(key, 0..usize::MAX).await
),
Err(err) => {
store.failed(index, err);
crate::sql_backend!(
&store.primary,
db => db.get_blob(key, 0..usize::MAX).await
)
}
},
None => crate::sql_backend!(
&store.primary,
db => db.get_blob(key, 0..usize::MAX).await
),
},
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
@@ -171,6 +198,9 @@ impl BlobStore {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.put_blob(key, data).await,
Store::Ephemeral(store) => store.put_blob(key, data).await,
Store::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.put_blob(key, data).await)
}
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.put_blob(key, data).await,
@@ -197,6 +227,9 @@ impl BlobStore {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.delete_blob(key).await,
Store::Ephemeral(store) => store.delete_blob(key).await,
Store::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.delete_blob(key).await)
}
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.delete_blob(key).await,
+2
View File
@@ -26,6 +26,8 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(_) => "rocksdb",
Self::Ephemeral(_) => "ephemeral",
// inbuxa: ST-3: as its primary
Self::Replicated(store) => store.primary.id(),
Self::None => "none",
}
}
+45
View File
@@ -201,6 +201,25 @@ impl SearchStore {
filters: &[SearchFilter],
sort: &[SearchComparator],
) -> trc::Result<Vec<u32>> {
// inbuxa: ST-6, ST-12: a replica answers search queries in a read scope
if let SearchStore::Store(Store::Replicated(store)) = self {
return match store.read_target(crate::SUBSPACE_SEARCH_INDEX).await {
Some(replica) => match crate::sql_backend!(
&store.replicas[replica].store,
db => db.query(index, filters, sort).await
) {
Ok(ids) => {
store.served(replica);
Ok(ids)
}
Err(err) => {
store.failed(replica, err);
crate::sql_backend!(&store.primary, db => db.query(index, filters, sort).await)
}
},
None => crate::sql_backend!(&store.primary, db => db.query(index, filters, sort).await),
};
}
match self {
SearchStore::Store(store) => match store {
#[cfg(feature = "postgres")]
@@ -215,6 +234,13 @@ impl SearchStore {
}
pub async fn query_global(&self, query: SearchQuery) -> trc::Result<Vec<u64>> {
// inbuxa: ST-5: global queries are maintenance, on the primary
if let SearchStore::Store(Store::Replicated(store)) = self {
return crate::sql_backend!(
&store.primary,
db => db.query(query.index, &query.filters, &query.comparators).await
);
}
match self {
SearchStore::Store(store) => match store {
#[cfg(feature = "postgres")]
@@ -245,6 +271,9 @@ impl SearchStore {
}
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
if let SearchStore::Store(Store::Replicated(store)) = self {
return crate::sql_backend!(&store.primary, db => db.index(documents).await);
}
match self {
SearchStore::Store(store) => match store {
#[cfg(feature = "postgres")]
@@ -259,6 +288,9 @@ impl SearchStore {
}
pub async fn unindex(&self, query: SearchQuery) -> trc::Result<u64> {
if let SearchStore::Store(Store::Replicated(store)) = self {
return crate::sql_backend!(&store.primary, db => db.unindex(query).await);
}
match self {
SearchStore::Store(store) => match store {
#[cfg(feature = "postgres")]
@@ -279,6 +311,8 @@ impl SearchStore {
Store::PostgreSQL(_) => None,
#[cfg(feature = "mysql")]
Store::MySQL(_) => None,
// inbuxa: ST-3: as its primary
Store::Replicated(replicated) if replicated.primary.is_pg_or_mysql() => None,
store => Some(store),
},
_ => None,
@@ -289,6 +323,10 @@ impl SearchStore {
match self {
#[cfg(feature = "mysql")]
SearchStore::Store(Store::MySQL(_)) => true,
#[cfg(feature = "mysql")]
SearchStore::Store(Store::Replicated(store)) => {
matches!(store.primary, Store::MySQL(_))
}
_ => false,
}
}
@@ -297,6 +335,10 @@ impl SearchStore {
match self {
#[cfg(feature = "postgres")]
SearchStore::Store(Store::PostgreSQL(_)) => true,
#[cfg(feature = "postgres")]
SearchStore::Store(Store::Replicated(store)) => {
matches!(store.primary, Store::PostgreSQL(_))
}
_ => false,
}
}
@@ -316,6 +358,9 @@ impl SearchStore {
Store::PostgreSQL(store) => store.create_search_tables().await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.create_search_tables().await,
Store::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.create_search_tables().await)
}
_ => Ok(()),
},
SearchStore::ElasticSearch(store) => store.create_indexes().await,
+103
View File
@@ -35,6 +35,23 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.get_value(key).await,
Self::Ephemeral(store) => store.get_value(key).await,
// inbuxa: ST-6 to ST-8, ST-12
Self::Replicated(store) => match store.read_target(key.subspace()).await {
Some(index) => {
match crate::sql_backend!(&store.replicas[index].store, db => db.get_value::<U>(key.clone()).await) {
Ok(Some(value)) => {
store.served(index);
Ok(Some(value))
}
Ok(None) => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.get_value(key).await)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
},
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -53,6 +70,21 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.key_exists(key).await,
Self::Ephemeral(store) => store.key_exists(key).await,
// inbuxa: ST-6 to ST-8, ST-12
Self::Replicated(store) => match store.read_target(key.subspace()).await {
Some(index) => match crate::sql_backend!(&store.replicas[index].store, db => db.key_exists(key.clone()).await) {
Ok(true) => {
store.served(index);
Ok(true)
}
Ok(false) => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.key_exists(key).await)
}
},
None => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
},
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -76,6 +108,38 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.iterate(params, cb).await,
Self::Ephemeral(store) => store.iterate(params, cb).await,
// inbuxa: ST-6, ST-12: a failed replica iteration is repeated on
// the primary when nothing was handed to the callback yet
#[allow(unused_mut, unused_variables)]
Self::Replicated(store) => {
let mut cb = cb;
match store.read_target(params.begin.subspace()).await {
Some(index) => {
let mut called = false;
let result = crate::sql_backend!(&store.replicas[index].store, db => db
.iterate(params.clone(), |key, value| {
called = true;
cb(key, value)
})
.await);
match result {
Ok(()) => {
store.served(index);
Ok(())
}
Err(err) if !called => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await)
}
Err(err) => {
store.failed(index, err.clone());
Err(err)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await),
}
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!());
@@ -104,6 +168,25 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.get_counter(key).await,
Self::Ephemeral(store) => store.get_counter(key).await,
// inbuxa: ST-6, ST-12
Self::Replicated(store) => {
let key: ValueKey<ValueClass> = key.into();
match store.read_target(crate::Key::subspace(&key)).await {
Some(index) => {
match crate::sql_backend!(&store.replicas[index].store, db => db.get_counter(key.clone()).await) {
Ok(value) => {
store.served(index);
Ok(value)
}
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.get_counter(key).await)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.get_counter(key).await),
}
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -123,6 +206,10 @@ impl Store {
Self::PostgreSQL(store) => store.sql_query(query, &params).await,
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.sql_query(query, &params).await,
// inbuxa: ST-5: operator-written statements always go to the primary
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.sql_query(query, &params).await)
}
_ => Err(trc::StoreEvent::NotSupported.into_err()),
};
@@ -152,6 +239,15 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.write(batch).await,
Self::Ephemeral(store) => store.write(batch).await,
// inbuxa: ST-5, ST-7: writes go to the primary, and record marks
Self::Replicated(store) => {
crate::backend::scaleout::replica::note_scope_write();
let result = crate::sql_backend!(&store.primary, db => db.write(batch).await);
if let Ok(ids) = &result {
store.note_write(ids).await;
}
result
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
};
@@ -197,6 +293,7 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.purge_store().await,
Self::Ephemeral(store) => store.purge_store().await,
Self::Replicated(store) => crate::sql_backend!(&store.primary, db => db.purge_store().await),
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -215,6 +312,9 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.delete_range(from, to).await,
Self::Ephemeral(store) => store.delete_range(from, to).await,
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.delete_range(from, to).await)
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -360,6 +460,9 @@ impl Store {
Self::PostgreSQL(store) => store.create_storage_tables().await,
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.create_storage_tables().await,
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.create_storage_tables().await)
}
_ => Ok(()),
}
}
+9 -2
View File
@@ -159,6 +159,8 @@ pub enum Store {
#[cfg(feature = "rocks")]
RocksDb(Arc<backend::rocksdb::RocksDbStore>),
Ephemeral(Arc<EphemeralStore>),
// inbuxa: ST-5 to ST-15: a PostgreSQL or MySQL primary with read replicas
Replicated(Arc<backend::scaleout::replica::ReplicatedStore>),
#[default]
None,
}
@@ -654,8 +656,8 @@ impl Store {
#[cfg(feature = "rocks")]
(Store::RocksDb(a), Store::RocksDb(b)) => Arc::ptr_eq(a, b),
(Store::Ephemeral(a), Store::Ephemeral(b)) => Arc::ptr_eq(a, b),
#[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))]
(Store::SQLReadReplica(a), Store::SQLReadReplica(b)) => Arc::ptr_eq(a, b),
// inbuxa: ST-3
(Store::Replicated(a), Store::Replicated(b)) => Arc::ptr_eq(a, b),
(Store::None, Store::None) => true,
_ => false,
}
@@ -664,6 +666,8 @@ impl Store {
#[inline(always)]
pub fn is_sql(&self) -> bool {
match self {
// inbuxa: ST-3: as its primary
Store::Replicated(store) => store.primary.is_sql(),
#[cfg(feature = "sqlite")]
Store::SQLite(_) => true,
#[cfg(feature = "postgres")]
@@ -677,6 +681,8 @@ impl Store {
#[inline(always)]
pub fn is_pg_or_mysql(&self) -> bool {
match self {
// inbuxa: ST-3: as its primary
Store::Replicated(store) => store.primary.is_pg_or_mysql(),
#[cfg(feature = "mysql")]
Store::MySQL(_) => true,
#[cfg(feature = "postgres")]
@@ -715,6 +721,7 @@ impl std::fmt::Debug for Store {
#[cfg(feature = "rocks")]
Self::RocksDb(_) => f.debug_tuple("RocksDb").finish(),
Self::Ephemeral(_) => f.debug_tuple("Ephemeral").finish(),
Self::Replicated(store) => f.debug_tuple("Replicated").field(&store.primary).finish(),
Self::None => f.debug_tuple("None").finish(),
}