mspinola · mspinola · Aug 21, 2026 · Aug 21, 2026
diff --git a/docs/SYNCING.md b/docs/SYNCING.md
diff --git a/docs/WINDOWS_SCHEDULING.md b/docs/WINDOWS_SCHEDULING.md
@@ -28,7 +28,7 @@ Create **two** wrapper scripts — they run *different* commands from *different
 
 > **Ready-made templates:** copy [`docs/examples/windows/run-prices.cmd`](examples/windows/run-prices.cmd) and [`run-cot.cmd`](examples/windows/run-cot.cmd) out of the repo into your `<DIR>` (e.g. `C:\Users\you\cotdata\scheduler\`) rather than retyping them — then just fill in the placeholders. Keep them outside the repo so a `git pull` never clobbers your edited paths.
 
-> **Fill in your real paths.** Inside the `.cmd` files, overwrite the plain-text markers `REPLACE_WITH_STORE_PATH` (your synced store, e.g. `\\Mac\code\cotdata_store`) and `REPLACE_WITH_VENV_PATH` (your virtualenv, e.g. `C:\Users\you\code\cotdata\.venv`). **Don't use angle-bracket placeholders like `<STORE>` inside a `.cmd`** — cmd reads `<` and `>` as redirection and the script fails with "The syntax of the command is incorrect," even on `REM` comment lines. The `<DIR>` notation in the *task commands* further down is fine to substitute since those are quoted or typed at the prompt.
+> **Fill in your real paths.** Inside the `.cmd` files, overwrite the plain-text markers — `REPLACE_WITH_STORE_PATH` (your synced store, e.g. `\\Mac\code\cotdata_store`) and `REPLACE_WITH_VENV_PATH` (your virtualenv, e.g. `C:\Users\you\code\cotdata\.venv`) in the producer wrappers, and the per-store `REPLACE_WITH_COTDATA_*` / `REPLACE_WITH_MARKETDATA_*` pairs in [`sync-store.cmd`](examples/windows/sync-store.cmd) and [`push-to-server.cmd`](examples/windows/push-to-server.cmd), which mirror **two** stores since ADR-0007 (see [SYNCING.md](SYNCING.md)). No marker is a prefix of another, so a find-and-replace is safe. **Don't use angle-bracket placeholders like `<STORE>` inside a `.cmd`** — cmd reads `<` and `>` as redirection and the script fails with "The syntax of the command is incorrect," even on `REM` comment lines. The `<DIR>` notation in the *task commands* further down is fine to substitute since those are quoted or typed at the prompt.
 
 `run-prices.cmd` — bars (with `--require-final`, so it runs only once Norgate's **Final** prices are in, not interim bars). Note `MARKETDATA_STORE`: a *different* directory from `COTDATA_STORE`, not an alias for it.
 

diff --git a/docs/examples/mac/pull-store.sh b/docs/examples/mac/pull-store.sh
@@ -1,33 +1,55 @@
 #!/usr/bin/env bash
-# Pull the cotdata store from the producer machine onto a read-only replica.
+# Pull BOTH stores from the producer machine onto a read-only replica.
 # The consumer decides when to fetch, so this is the alternative to the producer
 # pushing (see examples/windows/sync-store.cmd).
 #
+# Since ADR-0007 the producer writes two stores and this script pulls both:
+#   COT store  CFTC positioning  (cotdata-cot --cot-all)
+#   bar store  bars + specs      (marketdata-update --bars / --metadata)
+# Separate passes with separate exclusions. They disagree about manifest.json --
+# legacy in one, the only index in the other -- so one list cannot serve both.
+#
 # Copy this next to your launchd/cron config and overwrite the markers below:
-#   REPLACE_WITH_PRODUCER_HOST = ssh target        e.g. matt@windows-box
-#   REPLACE_WITH_REMOTE_STORE  = store ON that host e.g. /c/Users/matt/cotdata_store
-#   REPLACE_WITH_LOCAL_STORE   = store on THIS mac  e.g. /Users/you/code/cotdata_store
+#   REPLACE_WITH_PRODUCER_HOST           = ssh target  e.g. matt@windows-box
+#   REPLACE_WITH_REMOTE_COTDATA_STORE    = COT store ON that host  e.g. /c/Users/matt/cotdata_store
+#   REPLACE_WITH_LOCAL_COTDATA_STORE     = COT store on THIS mac   e.g. /Users/you/code/cotdata_store
+#   REPLACE_WITH_REMOTE_MARKETDATA_STORE = bar store ON that host  e.g. /c/Users/matt/code/marketdata_store
+#   REPLACE_WITH_LOCAL_MARKETDATA_STORE  = bar store on THIS mac   e.g. /Users/you/code/marketdata_store
+# The marker names match verify-replicas.sh deliberately, and none is a prefix of
+# another: a find-and-replace over the shorter name would otherwise mangle the longer.
 # (Plain-text markers, not angle-bracket placeholders: an unedited <...> would be
 # read as a shell redirection.)
 #
 # rsync must exist on BOTH ends. On Windows that means WSL or a packaged rsync. With
 # only OpenSSH, `scp -r` works and the payload is small, but you lose deletion
 # handling. See docs/SYNCING.md.
-set -euo pipefail
+#
+# Note there is no `set -e`. Both stores are pulled even when the first fails: they
+# are independent, and aborting early would let a COT hiccup silently stop bars
+# reaching this Mac for as long as it lasted. The exit code below still reports it.
+set -uo pipefail
+
+HOST="REPLACE_WITH_PRODUCER_HOST"
+COT_SRC="$HOST:REPLACE_WITH_REMOTE_COTDATA_STORE"
+COT_DEST="REPLACE_WITH_LOCAL_COTDATA_STORE"
+BAR_SRC="$HOST:REPLACE_WITH_REMOTE_MARKETDATA_STORE"
+BAR_DEST="REPLACE_WITH_LOCAL_MARKETDATA_STORE"
 
-SRC="REPLACE_WITH_PRODUCER_HOST:REPLACE_WITH_REMOTE_STORE"
-DEST="REPLACE_WITH_LOCAL_STORE"
+rc=0
 
+# ── COT store ───────────────────────────────────────────────────────────────
 # What each exclusion is for:
-#   _cache, _raw   producer-internal, most of the bytes. _cache is cotdata's cache
-#                  of downloaded CFTC zips. _raw is a pre-ADR-0007 leftover (databento's
-#                  paid bronze store, now marketdata's) — still excluded.
+#   _cache         cotdata's cache of downloaded CFTC zips: producer-internal and
+#                  free to rebuild, and most of the bytes.
+#   _raw           pre-ADR-0007 leftover (databento's paid bronze store, now
+#                  marketdata's). Kept so a store built before the move still
+#                  excludes it.
 #   citpy          consumer-owned, not written by any producer, so --delete removes
 #                  it and no producer run brings it back. Kept as a backstop: such
 #                  files belong outside the store. See docs/SYNCING.md.
 #   manifest.json  legacy aggregate, nothing writes it, and it is the one file a
 #                  sync resolves last-writer-wins across both producer halves
-EXCLUDES=(
+COT_EXCLUDES=(
   --exclude '_cache/'
   --exclude '_raw/'
   --exclude 'citpy/'
@@ -36,11 +58,38 @@ EXCLUDES=(
 
 # Two passes so a manifest never arrives before the data it describes. Harmless if
 # reversed (readers open parquet directly), but free to get right.
-rsync -az --delete "${EXCLUDES[@]}" --exclude 'manifests/' "$SRC/" "$DEST/"
-rsync -az "$SRC/manifests/" "$DEST/manifests/"
-
-# Confirm what landed. Compare against the producer's own --check output: the lag
-# column is measured on WRITE time, so an entry the producer skipped shows as behind
-# even when its data looks fine.
-export COTDATA_STORE="$DEST"
-cotdata-update --check
+rsync -az --delete "${COT_EXCLUDES[@]}" --exclude 'manifests/' "$COT_SRC/" "$COT_DEST/" || rc=$?
+rsync -az "$COT_SRC/manifests/" "$COT_DEST/manifests/" || rc=$?
+
+# ── bar store ───────────────────────────────────────────────────────────────
+# A DIFFERENT list, and the difference is load-bearing:
+#
+#   manifest.json is the bar store's ONLY index -- marketdata keeps one file at the
+#   store root, not a manifests/ directory. It is held out of the --delete pass so
+#   the local copy is not removed before the new one lands, then pulled on its own
+#   line. Reusing COT_EXCLUDES here would have excluded it outright, leaving a bar
+#   store this Mac cannot enumerate. rsync --exclude matches by NAME AT ANY DEPTH,
+#   the same trap docs/SYNCING.md documents for vintage/snapshots.json.
+#
+#   _cache and citpy are absent from the bar store: no marketdata provider writes a
+#   download cache, and citpy is a cotdata-store consumer artefact. _raw IS
+#   excluded -- databento's append-only PAID raw store
+#   ($MARKETDATA_DATABENTO_RAW, else _raw/databento under the bar store) -- because
+#   a replica has no use for it and re-fetching costs money (ADR-0006).
+BAR_EXCLUDES=(
+  --exclude '_raw/'
+)
+
+rsync -az --delete "${BAR_EXCLUDES[@]}" --exclude 'manifest.json' "$BAR_SRC/" "$BAR_DEST/" || rc=$?
+rsync -az "$BAR_SRC/manifest.json" "$BAR_DEST/manifest.json" || rc=$?
+
+# Confirm what landed, per store. Compare against the producer's own --check
+# output: cotdata's lag column is measured on WRITE time, so an entry the producer
+# skipped shows as behind even when its data looks fine.
+COTDATA_STORE="$COT_DEST" cotdata-update --check || rc=$?
+echo
+# `marketdata-update --check` exits 1 on an EMPTY store, which is the failure this
+# whole change exists to catch: bars that never arrive at all.
+MARKETDATA_STORE="$BAR_DEST" marketdata-update --check || rc=$?
+
+exit "$rc"
diff --git a/docs/examples/mac/verify-replicas.sh b/docs/examples/mac/verify-replicas.sh
@@ -1,54 +1,127 @@
 #!/usr/bin/env bash
-# verify-replicas.sh — confirm the latest producer run reached BOTH store replicas
-# TODAY: one local store and one remote store read over SSH.
+# verify-replicas.sh — confirm the latest producer run reached BOTH replicas and
+# BOTH stores: one local (Mac, over SMB) and one remote (dash VPS, over SSH), each
+# holding a COT store and a bar store since ADR-0007.
 #
-# How it works: the producer rewrites `status.json` on every push, and the sync
-# carries it, so if a replica's status.json mtime date == the day this runs, that
-# replica received today's push. Run it AFTER the producer's scheduled run, e.g.
-# from launchd/cron. Exits 0 on PASS (both current), 1 on FAIL (names the laggard).
+# Four checks, not two. Bars moved to their own store with their own sync pass, so
+# a COT-only check would pass green while no bar has reached a replica in weeks --
+# which is exactly the failure this file was extended to catch.
+#
+# Run it AFTER the producer's scheduled run, e.g. from launchd/cron.
+# Exits 0 on PASS, 1 on FAIL (naming every laggard, not just the first).
+#
+# ── The two stores need two different freshness signals ─────────────────────
+# COT store: cotdata rewrites `status.json` on EVERY run, new data or not. So
+#   "status.json mtime is today" is a clean did-this-replica-update-today signal.
+# Bar store: marketdata has no status.json, and it rewrites `manifest.json` only
+#   when a bar or a spec is actually WRITTEN. A weekend, a holiday, or a deferred
+#   `--require-final` run legitimately writes nothing, so demanding "today" here
+#   would fail every Saturday. The check is a staleness WINDOW instead
+#   (BAR_MAX_AGE_DAYS, default 4 — Friday's write is still fresh on Tuesday).
+# Both syncs preserve timestamps (rsync -a, robocopy), so a replica's mtime is the
+# PRODUCER's write time, not the copy time. That is what makes either check mean
+# anything, and it is what makes the cross-check at the end possible.
 #
 # Configure by editing the block below or exporting the vars before calling.
 # Markers are plain text (not <angle brackets>) so an unedited copy still parses.
 set -uo pipefail
 
 # ── config ──────────────────────────────────────────────────────────────────
-LOCAL_STORE="${LOCAL_STORE:-REPLACE_WITH_LOCAL_STORE}"             # e.g. $HOME/code/cotdata_store
-LOCAL_CHECK="${LOCAL_CHECK:-REPLACE_WITH_LOCAL_COTDATA_UPDATE}"    # cotdata-update path, e.g. .../.venv/bin/cotdata-update
-REMOTE="${REMOTE:-REPLACE_WITH_REMOTE}"                            # ssh target, e.g. deploy@dash.example.com
-REMOTE_STORE="${REMOTE_STORE:-REPLACE_WITH_REMOTE_STORE}"          # e.g. /srv/cotdata_store
-REMOTE_CHECK="${REMOTE_CHECK:-REPLACE_WITH_REMOTE_COTDATA_UPDATE}" # cotdata-update path ON the remote
-SSH_KEY="${SSH_KEY:-}"                                             # optional: private key; empty = default key/agent
+LOCAL_COT="${LOCAL_COT:-REPLACE_WITH_LOCAL_COTDATA_STORE}"          # e.g. $HOME/code/cotdata_store
+LOCAL_BARS="${LOCAL_BARS:-REPLACE_WITH_LOCAL_MARKETDATA_STORE}"     # e.g. $HOME/code/marketdata_store
+LOCAL_COT_CHECK="${LOCAL_COT_CHECK:-REPLACE_WITH_LOCAL_COTDATA_UPDATE}"       # .../.venv/bin/cotdata-update
+LOCAL_BAR_CHECK="${LOCAL_BAR_CHECK:-REPLACE_WITH_LOCAL_MARKETDATA_UPDATE}"    # .../.venv/bin/marketdata-update
+REMOTE="${REMOTE:-REPLACE_WITH_REMOTE_SSH_TARGET}"                             # ssh target, e.g. deploy@dash.example.com
+REMOTE_COT="${REMOTE_COT:-REPLACE_WITH_REMOTE_COTDATA_STORE}"       # e.g. /srv/cotdata_store
+REMOTE_BARS="${REMOTE_BARS:-REPLACE_WITH_REMOTE_MARKETDATA_STORE}"  # e.g. /srv/marketdata_store
+REMOTE_COT_CHECK="${REMOTE_COT_CHECK:-REPLACE_WITH_REMOTE_COTDATA_UPDATE}"    # cotdata-update ON the remote
+REMOTE_BAR_CHECK="${REMOTE_BAR_CHECK:-REPLACE_WITH_REMOTE_MARKETDATA_UPDATE}" # marketdata-update ON the remote
+BAR_MAX_AGE_DAYS="${BAR_MAX_AGE_DAYS:-4}"                           # see the note above
+SSH_KEY="${SSH_KEY:-}"                                              # optional: private key; empty = default key/agent
 # ────────────────────────────────────────────────────────────────────────────
 
 TODAY=$(date +%F)
+NOW=$(date +%s)
 SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8)
 [ -n "$SSH_KEY" ] && SSH_OPTS+=(-i "$SSH_KEY")
 
 pass=0; fail=0
-check_date() {  # $1 = status.json mtime date (YYYY-MM-DD), $2 = label
-  if [ "$1" = "$TODAY" ]; then
-    echo "  PASS: $2 status.json written today ($1)"; pass=$((pass + 1))
+
+# date -r <file> prints the file's mtime on both macOS (BSD) and Linux (GNU).
+mtime_local()  { date -r "$1" +%s 2>/dev/null; }
+mtime_remote() { ssh "${SSH_OPTS[@]}" "$REMOTE" "date -r '$1' +%s" 2>/dev/null; }
+
+# Format an EPOCH, and note this is a different flag on each platform: BSD reads
+# `date -r` as either a file or a seconds count, GNU reads it as a file only and
+# wants `date -d @seconds` for the number. This file lives under mac/ but the
+# fallback costs one line and stops it breaking the day someone runs it on Linux.
+fmt_epoch() { date -r "$1" "${2:-+%F}" 2>/dev/null || date -d "@$1" "${2:-+%F}" 2>/dev/null; }
+
+check_today() {  # $1 = mtime epoch (may be empty), $2 = label
+  local when; when=$([ -n "$1" ] && fmt_epoch "$1" || echo "")
+  if [ "$when" = "$TODAY" ]; then
+    echo "  PASS: $2 status.json written today ($when)"; pass=$((pass + 1))
+  else
+    echo "  FAIL: $2 status.json last written '${when:-unknown}', not today ($TODAY)"; fail=$((fail + 1))
+  fi
+}
+
+check_age() {  # $1 = mtime epoch (may be empty), $2 = label
+  if [ -z "$1" ]; then
+    echo "  FAIL: $2 manifest.json missing or unreadable — no bars have reached this replica"
+    fail=$((fail + 1)); return
+  fi
+  local age_days when
+  age_days=$(( (NOW - $1) / 86400 ))
+  when=$(fmt_epoch "$1")
+  if [ "$age_days" -le "$BAR_MAX_AGE_DAYS" ]; then
+    echo "  PASS: $2 manifest.json written $when (${age_days}d old, window ${BAR_MAX_AGE_DAYS}d)"
+    pass=$((pass + 1))
   else
-    echo "  FAIL: $2 status.json last written '${1:-unknown}', not today ($TODAY)"; fail=$((fail + 1))
+    echo "  FAIL: $2 manifest.json written $when — ${age_days}d old, past the ${BAR_MAX_AGE_DAYS}d window"
+    fail=$((fail + 1))
   fi
 }
 
-echo "=== cotdata replica verification — $TODAY ==="
+echo "=== store replica verification — $TODAY ==="
 echo
-echo "[local] $LOCAL_STORE"
-# date -r <file> prints the file's mtime on both macOS (BSD) and Linux (GNU).
-check_date "$(date -r "$LOCAL_STORE/status.json" +%F 2>/dev/null)" "local"
-COTDATA_STORE="$LOCAL_STORE" "$LOCAL_CHECK" --check 2>&1 | sed 's/^/    /'
+echo "[local COT]  $LOCAL_COT"
+check_today "$(mtime_local "$LOCAL_COT/status.json")" "local COT"
+COTDATA_STORE="$LOCAL_COT" "$LOCAL_COT_CHECK" --check 2>&1 | sed 's/^/    /'
 echo
-echo "[remote] $REMOTE:$REMOTE_STORE"
-check_date "$(ssh "${SSH_OPTS[@]}" "$REMOTE" "date -r '$REMOTE_STORE/status.json' +%F" 2>/dev/null)" "remote"
-ssh "${SSH_OPTS[@]}" "$REMOTE" "COTDATA_STORE=$REMOTE_STORE $REMOTE_CHECK --check" 2>&1 | sed 's/^/    /'
+echo "[local bars] $LOCAL_BARS"
+L_BAR_MTIME=$(mtime_local "$LOCAL_BARS/manifest.json")
+check_age "$L_BAR_MTIME" "local bar"
+# --check exits 1 on an EMPTY bar store, the failure this file exists to catch.
+MARKETDATA_STORE="$LOCAL_BARS" "$LOCAL_BAR_CHECK" --check 2>&1 | sed 's/^/    /'
 echo
+echo "[remote COT]  $REMOTE:$REMOTE_COT"
+check_today "$(mtime_remote "$REMOTE_COT/status.json")" "remote COT"
+ssh "${SSH_OPTS[@]}" "$REMOTE" "COTDATA_STORE=$REMOTE_COT $REMOTE_COT_CHECK --check" 2>&1 | sed 's/^/    /'
+echo
+echo "[remote bars] $REMOTE:$REMOTE_BARS"
+R_BAR_MTIME=$(mtime_remote "$REMOTE_BARS/manifest.json")
+check_age "$R_BAR_MTIME" "remote bar"
+ssh "${SSH_OPTS[@]}" "$REMOTE" "MARKETDATA_STORE=$REMOTE_BARS $REMOTE_BAR_CHECK --check" 2>&1 | sed 's/^/    /'
+echo
+
+# Cross-check, free and strictly sharper than either window on its own. Both syncs
+# preserve timestamps, so BOTH replicas should carry the producer's own mtime,
+# identical to the second. If they differ, one push is behind the other — and that
+# stays true inside the staleness window, where neither replica looks wrong alone.
+if [ -n "$L_BAR_MTIME" ] && [ -n "$R_BAR_MTIME" ] && [ "$L_BAR_MTIME" != "$R_BAR_MTIME" ]; then
+  echo "  WARN: the two bar replicas hold DIFFERENT manifest mtimes"
+  echo "          local  $(fmt_epoch "$L_BAR_MTIME" '+%F %T')"
+  echo "          remote $(fmt_epoch "$R_BAR_MTIME" '+%F %T')"
+  echo "        One of the two bar pushes is behind. Both syncs preserve timestamps,"
+  echo "        so a matching producer run should land the same mtime on both."
+  echo
+fi
 
-if [ "$fail" -eq 0 ] && [ "$pass" -eq 2 ]; then
-  echo "RESULT: PASS — both replicas received today's producer push."
+if [ "$fail" -eq 0 ] && [ "$pass" -eq 4 ]; then
+  echo "RESULT: PASS — both replicas hold a current COT store and a current bar store."
   exit 0
 else
-  echo "RESULT: FAIL — $fail replica(s) did not update today. Check the producer task and its chained sync."
+  echo "RESULT: FAIL — $fail of 4 checks did not pass. Check the producer tasks and their chained syncs."
   exit 1
 fi