From 2c92d7039269c56e76f7072474eef361db84e0c0 Mon Sep 17 00:00:00 2001 From: Nick Potts Date: Sat, 31 Oct 2026 00:19:20 +0000 Subject: [PATCH 1/4] Sync each Corrosion commit with synchronous=full instead of making every store write synchronous The store's inode sync flag synced every page, so a healing Machine held Corrosion's one write connection for up to 51 s per commit and its own writes passed ployzd's 10 s limit. FULL syncs the WAL once per commit, before Corrosion broadcasts it, which keeps the power-cut guarantee. Stores an older daemon flagged are cleared at start. Machines now report store_syncs_commits, so the CLI warns about ZFS and Btrfs stores only on daemons that predate this. --- core/crates/ployz-core/src/domain/machine.rs | 5 + core/crates/ployz-core/tests/runtime_watch.rs | 1 + core/crates/ployz/src/handlers/server/show.rs | 18 ++- core/crates/ployzd/src/corrosion/service.rs | 124 +++++++++--------- core/crates/ployzd/src/machine/mod.rs | 5 +- core/crates/ployzd/tests/machine_record.rs | 1 + docs/user/troubleshooting/deployments.md | 7 +- 7 files changed, 94 insertions(+), 67 deletions(-) diff --git a/core/crates/ployz-core/src/domain/machine.rs b/core/crates/ployz-core/src/domain/machine.rs index f53095dbe3..f14acf8b81 100644 --- a/core/crates/ployz-core/src/domain/machine.rs +++ b/core/crates/ployz-core/src/domain/machine.rs @@ -149,6 +149,11 @@ pub struct MachineRuntime { /// and from older daemons. #[ts(optional = nullable)] pub store_filesystem: Option, + /// Whether the replicated store syncs each commit to disk before peers see it, so a + /// power cut loses none of this Machine's changes. Absent from older daemons, so + /// absence claims nothing. + #[ts(optional = nullable)] + pub store_syncs_commits: Option, /// Every capability name the daemon binary serves. Empty from a daemon /// that predates the field, and when an older entry relays a newer peer's /// record, so empty claims nothing. diff --git a/core/crates/ployz-core/tests/runtime_watch.rs b/core/crates/ployz-core/tests/runtime_watch.rs index 4d98003777..f835d79a06 100644 --- a/core/crates/ployz-core/tests/runtime_watch.rs +++ b/core/crates/ployz-core/tests/runtime_watch.rs @@ -249,6 +249,7 @@ fn expected_frame() -> RuntimeWatchFrame { pool_committable_bytes: None, sealing: None, store_filesystem: None, + store_syncs_commits: None, capabilities: BTreeSet::new(), management_identity: None, leaving: false, diff --git a/core/crates/ployz/src/handlers/server/show.rs b/core/crates/ployz/src/handlers/server/show.rs index d1c8444bdc..8b2ddf090f 100644 --- a/core/crates/ployz/src/handlers/server/show.rs +++ b/core/crates/ployz/src/handlers/server/show.rs @@ -123,12 +123,16 @@ fn daemon_skew_warning(machines: &[MachineObservation], cli_version: &str) -> Op } } -/// Filesystems that refuse, or don't pass on, the flag that makes the store write -/// synchronously, by the name the daemon reports and the name a person knows. +/// Filesystems that refuse, or don't pass on, the flag that made the store of a daemon +/// up to 0.3.11 write synchronously, by the name the daemon reports and the name a +/// person knows. A daemon that reports its store syncs each commit is not among them. const POWER_CUT_STORES: [(&str, &str); 3] = [("zfs", "ZFS"), ("btrfs", "Btrfs"), ("tmpfs", "tmpfs")]; fn power_cut_store(runtime: &MachineRuntime) -> Option<&'static str> { + if runtime.store_syncs_commits == Some(true) { + return None; + } let filesystem = runtime.store_filesystem.as_deref()?; POWER_CUT_STORES .iter() @@ -854,6 +858,16 @@ mod tests { ); } + #[test] + fn a_server_whose_store_syncs_each_commit_is_not_warned_about_on_any_filesystem() { + let mut zfs = on("web-1", Some("zfs")); + zfs.machine.runtime.store_syncs_commits = Some(true); + assert_eq!(power_cut_warning(&[zfs]), None); + let mut tmpfs = on("web-2", Some("tmpfs")); + tmpfs.machine.runtime.store_syncs_commits = Some(false); + assert!(power_cut_warning(&[tmpfs]).is_some()); + } + #[test] fn one_server_that_can_lose_its_last_changes_is_named() { assert_eq!( diff --git a/core/crates/ployzd/src/corrosion/service.rs b/core/crates/ployzd/src/corrosion/service.rs index a1cf2cce26..cb747edb80 100644 --- a/core/crates/ployzd/src/corrosion/service.rs +++ b/core/crates/ployzd/src/corrosion/service.rs @@ -155,13 +155,13 @@ impl CorrosionConfig { fn install(&self) -> Result { create_private_dir(&self.paths.data_dir)?; - let refused = sync_writes(&self.paths.data_dir); + let refused = clear_sync_writes(&self.paths.data_dir); if let Some(first) = refused.first() { tracing::warn!( data_dir = %self.paths.data_dir.display(), refused = refused.len(), error = %ployz_core::error_chain::inline(first), - "Corrosion's commits can be lost on power loss: files in its data directory do not write synchronously" + "Corrosion's commits stay slow: files in its data directory still write every page synchronously" ); } create_private_dir(&self.paths.run_dir)?; @@ -173,6 +173,7 @@ impl CorrosionConfig { db: DbConfig { path: self.paths.data_dir.join("store.db"), schema_paths: vec![schema_path], + synchronous: "full", }, gossip: GossipConfig { addr: self.gossip_address, @@ -422,23 +423,29 @@ fn create_private_dir(path: &Path) -> Result<(), Error> { } #[derive(Debug, thiserror::Error)] -#[error("could not make {} write synchronously", path.display())] +#[error("could not stop {} writing every page synchronously", path.display())] struct SyncWritesError { path: PathBuf, source: std::io::Error, } -// Corrosion broadcasts a commit before SQLite syncs it, and peers never send a Machine its -// own changes back, so a commit lost to power loss after broadcast is never recovered. -fn sync_writes(dir: &Path) -> Vec { +// Daemons up to 0.3.11 set the inode sync flag on the store so a broadcast commit survived +// power loss. `synchronous = "full"` now syncs each commit, and the flag would sync every +// page on top, which held Corrosion's one write connection for minutes on a slow disk (#2925). +fn clear_sync_writes(dir: &Path) -> Vec { let mut refused = Vec::new(); sync_tree(dir, OFlags::DIRECTORY, &mut refused); - refused.retain(|error| error.source.kind() != std::io::ErrorKind::NotFound); + let unsupported = [rustix::io::Errno::OPNOTSUPP, rustix::io::Errno::NOTTY] + .map(|errno| Some(errno.raw_os_error())); + refused.retain(|error| { + error.source.kind() != std::io::ErrorKind::NotFound + && !unsupported.contains(&error.source.raw_os_error()) + }); refused } fn sync_tree(dir: &Path, open: OFlags, refused: &mut Vec) { - if let Err(error) = set_sync_flag(dir, open) { + if let Err(error) = clear_sync_flag(dir, open) { refused.push(error); } let entries = match fs::read_dir(dir) { @@ -468,7 +475,7 @@ fn sync_tree(dir: &Path, open: OFlags, refused: &mut Vec) { sync_tree(&path, OFlags::DIRECTORY | OFlags::NOFOLLOW, refused) } Ok(file_type) if file_type.is_file() => { - if let Err(error) = set_sync_flag(&path, OFlags::NOFOLLOW) { + if let Err(error) = clear_sync_flag(&path, OFlags::NOFOLLOW) { refused.push(error); } } @@ -478,7 +485,7 @@ fn sync_tree(dir: &Path, open: OFlags, refused: &mut Vec) { } } -fn set_sync_flag(path: &Path, kind: OFlags) -> Result<(), SyncWritesError> { +fn clear_sync_flag(path: &Path, kind: OFlags) -> Result<(), SyncWritesError> { let set = || -> rustix::io::Result<()> { let fd = rustix::fs::open( path, @@ -486,8 +493,8 @@ fn set_sync_flag(path: &Path, kind: OFlags) -> Result<(), SyncWritesError> { Mode::empty(), )?; let flags = rustix::fs::ioctl_getflags(&fd)?; - if !flags.contains(IFlags::SYNC) { - rustix::fs::ioctl_setflags(&fd, flags | IFlags::SYNC)?; + if flags.contains(IFlags::SYNC) { + rustix::fs::ioctl_setflags(&fd, flags - IFlags::SYNC)?; } Ok(()) }; @@ -543,6 +550,7 @@ struct FileConfig { struct DbConfig { path: PathBuf, schema_paths: Vec, + synchronous: &'static str, } #[derive(Serialize)] @@ -702,73 +710,67 @@ mod tests { assert_eq!(fs::metadata(&path).unwrap().mode() & 0o777, 0o600); } + fn writes_synchronously(path: &Path) -> bool { + rustix::fs::ioctl_getflags(fs::File::open(path).unwrap()) + .unwrap() + .contains(IFlags::SYNC) + } + + /// Flags `paths` the way daemons up to 0.3.11 did, or returns false where the + /// filesystem refuses the flag, as tmpfs and ZFS do. + fn flag_as_an_older_daemon(paths: &[PathBuf]) -> bool { + paths.iter().all(|path| { + let file = fs::File::open(path).unwrap(); + rustix::fs::ioctl_getflags(&file) + .and_then(|flags| rustix::fs::ioctl_setflags(&file, flags | IFlags::SYNC)) + .is_ok() + }) + } + #[test] - fn existing_store_files_write_synchronously() { + fn a_store_an_older_daemon_flagged_stops_writing_every_page_synchronously() { let root = tempfile::tempdir().unwrap(); let dir = root.path(); - fs::create_dir(dir.join("snapshots")).unwrap(); - fs::write(dir.join("snapshots/old.db"), "").unwrap(); + fs::create_dir(dir.join("subscriptions")).unwrap(); + fs::write(dir.join("subscriptions/sub.sqlite"), "").unwrap(); fs::write(dir.join("store.db"), "").unwrap(); - - let refused = sync_writes(dir); - if refused.is_empty() { - for path in [ - dir.to_path_buf(), - dir.join("snapshots"), - dir.join("snapshots/old.db"), - dir.join("store.db"), - ] { - let file = fs::File::open(&path).unwrap(); - assert!( - rustix::fs::ioctl_getflags(&file) - .unwrap() - .contains(IFlags::SYNC), - "{} does not write synchronously", - path.display() - ); - } - } else { - let unsupported = [rustix::io::Errno::OPNOTSUPP, rustix::io::Errno::NOTTY] - .map(|errno| Some(errno.raw_os_error())); - assert!(refused.iter().any(|error| error.path == dir)); - for error in &refused { - assert!( - unsupported.contains(&error.source.raw_os_error()), - "only the filesystem refusing the flag may leave files unflagged: {error}: {}", - error.source - ); + let paths = [ + dir.to_path_buf(), + dir.join("subscriptions"), + dir.join("subscriptions/sub.sqlite"), + dir.join("store.db"), + ]; + let flagged = flag_as_an_older_daemon(&paths); + + let refused = clear_sync_writes(dir); + + assert!(refused.is_empty(), "{refused:?}"); + if flagged { + for path in &paths { + assert!(!writes_synchronously(path), "{}", path.display()); } } } #[test] - fn one_refused_file_leaves_the_rest_of_the_store_writing_synchronously() { + fn one_unreadable_file_leaves_the_rest_of_the_store_cleared() { let root = tempfile::tempdir().unwrap(); let dir = root.path(); - let refused = dir.join("refused"); - fs::write(&refused, "").unwrap(); - fs::set_permissions(&refused, fs::Permissions::from_mode(0o000)).unwrap(); let siblings: Vec<_> = (0..20).map(|n| dir.join(format!("store-{n}.db"))).collect(); for sibling in &siblings { fs::write(sibling, "").unwrap(); } - - let _ = sync_writes(dir); - - let synchronous = |path: &Path| { - rustix::fs::ioctl_getflags(fs::File::open(path).unwrap()) - .unwrap() - .contains(IFlags::SYNC) - }; - if !synchronous(dir) { + if !flag_as_an_older_daemon(&siblings) { return; } + let unreadable = dir.join("unreadable"); + fs::write(&unreadable, "").unwrap(); + fs::set_permissions(&unreadable, fs::Permissions::from_mode(0o000)).unwrap(); + + let _ = clear_sync_writes(dir); + for sibling in &siblings { - assert!( - synchronous(sibling), - "{} does not write synchronously", - sibling.display() - ); + assert!(!writes_synchronously(sibling), "{}", sibling.display()); } } diff --git a/core/crates/ployzd/src/machine/mod.rs b/core/crates/ployzd/src/machine/mod.rs index 0472ee7b24..48e1b48078 100644 --- a/core/crates/ployzd/src/machine/mod.rs +++ b/core/crates/ployzd/src/machine/mod.rs @@ -601,6 +601,7 @@ impl LocalMachineRecord { #[must_use] pub fn local_runtime(data_dir: &Path) -> MachineRuntime { + let store_filesystem = store_filesystem(data_dir); MachineRuntime { daemon_version: env!("CARGO_PKG_VERSION").into(), docker_version: docker_version(Path::new("docker"), DOCKER_VERSION_TIMEOUT), @@ -626,7 +627,9 @@ pub fn local_runtime(data_dir: &Path) -> MachineRuntime { running_builds: 0, pool_committable_bytes: None, sealing: None, - store_filesystem: store_filesystem(data_dir), + // Corrosion syncs each commit; on tmpfs a sync reaches only memory. + store_syncs_commits: store_filesystem.as_deref().map(|name| name != "tmpfs"), + store_filesystem, capabilities: ployz_core::served_capabilities(), management_identity: None, leaving: false, diff --git a/core/crates/ployzd/tests/machine_record.rs b/core/crates/ployzd/tests/machine_record.rs index 4846e2341b..231e8592f9 100644 --- a/core/crates/ployzd/tests/machine_record.rs +++ b/core/crates/ployzd/tests/machine_record.rs @@ -381,6 +381,7 @@ fn reopening_a_participating_machine_refreshes_runtime_metadata() { pool_committable_bytes: None, sealing: None, store_filesystem: None, + store_syncs_commits: None, capabilities: Default::default(), management_identity: None, leaving: false, diff --git a/docs/user/troubleshooting/deployments.md b/docs/user/troubleshooting/deployments.md index 557e153d2f..287b07beb1 100644 --- a/docs/user/troubleshooting/deployments.md +++ b/docs/user/troubleshooting/deployments.md @@ -398,9 +398,10 @@ The restart makes web-2 report its replicas again, and every server sends reques deploy or **Redeploy** with nothing changed doesn't help, because it leaves the replicas as they are. If some requests still reach web-2's old replicas a minute later, restart the service again. -A server whose root disk uses ZFS, Btrfs or tmpfs can still lose its last changes in a power cut. If that -happens, restart the services on it, as above. `ployz server ls` names each such server, and -`ployz server show` adds a `store` line that says so. +A server whose root disk uses tmpfs can still lose its last changes in a power cut, and so can a +server on ZFS or Btrfs that runs an older Ployz. If that happens, restart the services on it, as +above. `ployz server ls` names each such server, and `ployz server show` adds a `store` line that +says so. ## My deploy failed when a server lost power or its network From 45eaa153e2ff8fb8470965b8f3e98ba4c3816e3d Mon Sep 17 00:00:00 2001 From: Nick Potts Date: Sat, 31 Oct 2026 00:20:58 +0000 Subject: [PATCH 2/4] Compare every replicated row in the power-cut check Brings rows-agree.sh and rows_diff.py from #3214. Equal clocks can hide different values, so the check now waits for every Machine to hold the same rows in every replicated table. --- core/scripts/verify/power-cut-replication.sh | 44 +++++++------- core/scripts/verify/rows-agree.sh | 42 +++++++++++++ core/scripts/verify/rows_diff.py | 63 ++++++++++++++++++++ 3 files changed, 129 insertions(+), 20 deletions(-) create mode 100755 core/scripts/verify/rows-agree.sh create mode 100644 core/scripts/verify/rows_diff.py diff --git a/core/scripts/verify/power-cut-replication.sh b/core/scripts/verify/power-cut-replication.sh index 7d4690d23f..40364103bc 100755 --- a/core/scripts/verify/power-cut-replication.sh +++ b/core/scripts/verify/power-cut-replication.sh @@ -6,8 +6,11 @@ # issued and Caddy renders no route, so it first pins a self-signed certificate on each custom # domain. Cuts power to (default: the last one), brings it back, cuts it again # 8 s after boot so the cut lands on its own fresh Corrosion commits, then brings it back. -# 60 s later it fails when any Machine holds a different clock or value for one of 's -# container rows, or when any proxy routes an address on that is not an In copy there. +# With HOT_SERVICE=, the first cut instead lands the moment `service restart` of that +# Service returns, on the commits the restart just made. +# From 60 s after the last boot, it fails when the Machines still hold different rows +# CONVERGE_SECONDS (default 120) later (rows-agree.sh, which needs sqlite3 on every Machine), +# or when any proxy routes an address on that is not an In copy there. set -Eeuo pipefail trap 'echo "could not run: line $LINENO"; exit 2' ERR cd "$(dirname "$0")/../.." @@ -45,7 +48,18 @@ certify() { rm -rf "$dir" } +for m in $names; do + vc exec "$run" "$m" -- sh -c 'command -v sqlite3' >/dev/null || { echo "needs sqlite3 on $m"; exit 2; } +done +copies="select count(*) from containers where machine_id = '$target_id' and rotation = 'in' + and namespace != 'ployz-system'" +actor="select hex(site_id) from crsql_site_id where ordinal = 0" +[ "$(store "$target" "$copies")" -gt 0 ] || { echo "needs a Service copy on $target"; exit 2; } +actor_before=$(store "$target" "$actor") certify +if [ -n "${HOT_SERVICE:-}" ]; then + vc cli "$run" -- service restart "$HOT_SERVICE" >/dev/null +fi vc power off "$run" "$target" >/dev/null; down vc power on "$run" "$target" >/dev/null sleep 8 @@ -53,24 +67,14 @@ vc power off "$run" "$target" >/dev/null; down vc power on "$run" "$target" >/dev/null sleep 60 -clocks="select p.id||' '||c.col_name||' '||c.col_version||' '||c.db_version - from containers__crsql_clock c join containers__crsql_pks p on p.__crsql_key = c.key - join containers t on t.id = p.id where t.machine_id = '$target_id' order by 1" -contents="select id||' '||gen||' '||rotation||' '||coalesce(json_extract(container, '\$.address'), '') - from containers where machine_id = '$target_id' order by 1" failed=0 -for query in "$clocks" "$contents"; do - reference=$(store "$target" "$query") - [ -n "$reference" ] || { echo "FAIL: $target holds no rows for its own containers"; failed=1; } - for m in $names; do - [ "$m" = "$target" ] && continue - if ! diff <(echo "$reference") <(store "$m" "$query") >/dev/null; then - echo "FAIL: $m disagrees with $target about $target's containers:" - diff <(echo "$reference") <(store "$m" "$query") || true - failed=1 - fi - done -done +[ "$(store "$target" "$copies")" -gt 0 ] || { echo "FAIL: $target holds no Service copy after the cuts"; failed=1; } +if [ "$(store "$target" "$actor")" = "$actor_before" ]; then + echo "$target kept its Corrosion actor" +else + echo "$target moved to a new Corrosion actor" +fi +scripts/verify/rows-agree.sh "$run" "$target" || { status=$?; [ "$status" = 1 ] || exit "$status"; failed=1; } serving=$(store "$target" "select json_extract(container, '\$.address') from containers where machine_id = '$target_id' and rotation = 'in' and json_extract(container, '\$.address') is not null") @@ -87,5 +91,5 @@ for m in $names; do done done -[ "$failed" = 0 ] && echo "PASS: every Machine agrees on $target's containers and routes only its In copies" +[ "$failed" = 0 ] && echo "PASS: every Machine holds the same rows and routes only In copies on $target" exit "$failed" diff --git a/core/scripts/verify/rows-agree.sh b/core/scripts/verify/rows-agree.sh new file mode 100755 index 0000000000..e046da23bf --- /dev/null +++ b/core/scripts/verify/rows-agree.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Usage: scripts/verify/rows-agree.sh [reference machine] +# +# Waits up to CONVERGE_SECONDS (default 120) for every Machine to hold the same rows in every +# replicated table, every column compared, and fails with the differing rows when they never +# do. Equal clocks are not enough: a reused db_version can leave equal clocks on Machines that +# hold different values. Needs sqlite3 on every Machine. +set -Eeuo pipefail +trap 'echo "could not run: line $LINENO"; exit 2' ERR +cd "$(dirname "$0")/../.." +run=$1 +vc() { scripts/verify-cluster "$@"; } +reference=${2:-$(jq -r '.machines[-1].name' "$run")} +names=$(jq -r '.machines[].name' "$run") +for m in $names; do + vc exec "$run" "$m" -- sh -c 'command -v sqlite3' >/dev/null || { echo "needs sqlite3 on $m"; exit 2; } +done +tables="cluster key|machines id|containers id|certificates hostname|volumes machine_id, name|ingress_routes machine_id, service_id" +query="" +while IFS=' ' read -r table key; do + query="$query select '$table' as tbl, * from $table order by $key;" +done < <(tr '|' '\n' <<<"$tables") +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +deadline=$(( SECONDS + ${CONVERGE_SECONDS:-120} )) +while :; do + for m in $names; do + vc exec "$run" "$m" -- sqlite3 -readonly -json /var/lib/ployz/corrosion/store.db "$query" >"$dir/$m" + done + differing=$(for m in $names; do cmp -s "$dir/$reference" "$dir/$m" || echo "$m"; done) + if [ -z "$differing" ]; then + echo "rows agree on every Machine: $(grep -c '"tbl":' "$dir/$reference") rows" + exit 0 + fi + [ "$SECONDS" -lt "$deadline" ] || break + sleep 5 +done +for m in $differing; do + echo "FAIL: $m holds different rows from $reference after ${CONVERGE_SECONDS:-120}s:" + python3 -I scripts/verify/rows_diff.py "$dir/$reference" "$dir/$m" +done +exit 1 diff --git a/core/scripts/verify/rows_diff.py b/core/scripts/verify/rows_diff.py new file mode 100644 index 0000000000..8b12377bb5 --- /dev/null +++ b/core/scripts/verify/rows_diff.py @@ -0,0 +1,63 @@ +"""Print each row two `sqlite3 -json` dumps from rows-agree.sh disagree on, by field.""" + +import json +import sys + +KEYS = { + "cluster": ("key",), + "machines": ("id",), + "containers": ("id",), + "certificates": ("hostname",), + "volumes": ("machine_id", "name"), + "ingress_routes": ("machine_id", "service_id"), +} + + +def rows(path): + text, decoder, at, found = open(path).read(), json.JSONDecoder(), 0, {} + while at < len(text): + if text[at].isspace(): + at += 1 + continue + batch, at = decoder.raw_decode(text, at) + for row in batch: + row = {name.lower(): value for name, value in row.items()} + table = row["tbl"] + key = tuple(row[name] for name in KEYS[table]) + found[(table, key)] = row + return found + + +def leaves(value, path=""): + if isinstance(value, str) and value[:1] in "{[": + try: + value = json.loads(value) + except ValueError: + pass + if isinstance(value, dict): + for name, inner in value.items(): + yield from leaves(inner, f"{path}.{name}") + elif isinstance(value, list): + for index, inner in enumerate(value): + yield from leaves(inner, f"{path}[{index}]") + else: + yield path, value + + +def main(reference, other): + ours, theirs = rows(reference), rows(other) + for key in sorted(set(ours) | set(theirs), key=str): + mine, peer = ours.get(key), theirs.get(key) + if mine == peer: + continue + label = f" {key[0]} {'/'.join(map(str, key[1]))}" + if mine is None or peer is None: + print(f"{label}: only on {'the reference' if peer is None else 'this Machine'}") + continue + a, b = dict(leaves(mine)), dict(leaves(peer)) + for path in sorted(set(a) | set(b)): + if a.get(path) != b.get(path): + print(f"{label} {path}: reference {a.get(path)!r}, here {b.get(path)!r}") + + +main(*sys.argv[1:]) From 5a3013cf42ed143f3d8a19a524277954c8ee9757 Mon Sep 17 00:00:00 2001 From: Nick Potts Date: Sat, 31 Oct 2026 00:45:03 +0000 Subject: [PATCH 3/4] Pin the Corrosion image that reads [db] synchronous getployz/corrosion#1, built by the fork's image workflow at 28f00b417271. --- core/crates/ployz-testkit/Dockerfile | 2 +- core/crates/ployzd/src/corrosion/service.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/crates/ployz-testkit/Dockerfile b/core/crates/ployz-testkit/Dockerfile index 5421bb8c74..568a6f8fea 100644 --- a/core/crates/ployz-testkit/Dockerfile +++ b/core/crates/ployz-testkit/Dockerfile @@ -17,7 +17,7 @@ COPY --from=docker-bin /usr/local/libexec/docker/cli-plugins/docker-buildx /usr/ # The classic store cannot load an OCI archive; only priming, which runs on the # containerd store, loads these tars. RUN mkdir -p /opt/ployz/images \ - && skopeo copy --all docker://ghcr.io/getployz/corrosion@sha256:281efeb3ca82eac25e4ebb8a2cb6c61100a9db2948dcf507627aaa48e22e7dff oci-archive:/opt/ployz/images/corrosion.tar:ghcr.io/getployz/corrosion:20261024-b46434825343 \ + && skopeo copy --all docker://ghcr.io/getployz/corrosion@sha256:384f1e20dbbe187135c4c75ae0eb41943df64c2306b5b4eecffa88ba04b46cc1 oci-archive:/opt/ployz/images/corrosion.tar:ghcr.io/getployz/corrosion:20261031-28f00b417271 \ && skopeo copy docker://docker.io/library/alpine:3.23.3 docker-archive:/opt/ployz/images/alpine.tar:alpine:3.23.3 \ && skopeo copy docker://docker.io/library/caddy:2.10.2 docker-archive:/opt/ployz/images/caddy.tar:caddy:2.10.2 \ && skopeo copy docker://ghcr.io/psviderski/unregistry:0.4.1 docker-archive:/opt/ployz/images/unregistry.tar:ghcr.io/psviderski/unregistry:0.4.1 diff --git a/core/crates/ployzd/src/corrosion/service.rs b/core/crates/ployzd/src/corrosion/service.rs index cb747edb80..f4c50f974e 100644 --- a/core/crates/ployzd/src/corrosion/service.rs +++ b/core/crates/ployzd/src/corrosion/service.rs @@ -28,7 +28,7 @@ use ployz_core::{CORROSION_API_PORT, CORROSION_GOSSIP_PORT}; use super::{AdminClient, ApiClient, Error, ReplicatedStore, Statement}; /// Built from the `ployz` branch of getployz/corrosion: upstream plus the patches its README lists. -pub const IMAGE: &str = "ghcr.io/getployz/corrosion:20261024-b46434825343@sha256:281efeb3ca82eac25e4ebb8a2cb6c61100a9db2948dcf507627aaa48e22e7dff"; +pub const IMAGE: &str = "ghcr.io/getployz/corrosion:20261031-28f00b417271@sha256:384f1e20dbbe187135c4c75ae0eb41943df64c2306b5b4eecffa88ba04b46cc1"; pub const DEFAULT_CONTAINER_NAME: &str = "ployz-corrosion"; const TOKEN_FILE: &str = ".api-token"; const SCHEMA: &str = include_str!("schema.sql"); From bd2dd303b67ef587a11e2af620fe0701818ca253 Mon Sep 17 00:00:00 2001 From: Nick Potts Date: Sat, 31 Oct 2026 00:55:57 +0000 Subject: [PATCH 4/4] Clear the store's sync flag only once Corrosion syncs each commit itself Clearing it at install left an older Corrosion, still serving while the new image pulled, writing with neither the flag nor synchronous=full. A failed pull kept it that way. Found in review. --- core/crates/ployzd/src/corrosion/service.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/crates/ployzd/src/corrosion/service.rs b/core/crates/ployzd/src/corrosion/service.rs index f4c50f974e..6e0ac0cafa 100644 --- a/core/crates/ployzd/src/corrosion/service.rs +++ b/core/crates/ployzd/src/corrosion/service.rs @@ -125,6 +125,17 @@ impl CorrosionConfig { run_dir: self.paths.run_dir.clone(), }; let ensured = service.start(&files).await?; + // Only now does the running Corrosion sync each commit itself. An older one, + // still serving while the new image pulls, relies on the flag. + let refused = clear_sync_writes(&self.paths.data_dir); + if let Some(first) = refused.first() { + tracing::warn!( + data_dir = %self.paths.data_dir.display(), + refused = refused.len(), + error = %ployz_core::error_chain::inline(first), + "Corrosion's commits stay slow: files in its data directory still write every page synchronously" + ); + } wait_ready_or_replace( ensured == Ensure::Keep, || async { api.query(Statement::new("SELECT 1", [])).await.is_ok() }, @@ -155,15 +166,6 @@ impl CorrosionConfig { fn install(&self) -> Result { create_private_dir(&self.paths.data_dir)?; - let refused = clear_sync_writes(&self.paths.data_dir); - if let Some(first) = refused.first() { - tracing::warn!( - data_dir = %self.paths.data_dir.display(), - refused = refused.len(), - error = %ployz_core::error_chain::inline(first), - "Corrosion's commits stay slow: files in its data directory still write every page synchronously" - ); - } create_private_dir(&self.paths.run_dir)?; let token = load_or_create_token(&self.paths.token_file())?; let schema_path = self.paths.data_dir.join("schema.sql");