diff --git a/.gitignore b/.gitignore index ccc6329..a1c1102 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ dist/ # roll-rule investigation cache (scripts/investigate_databento_roll_rule.py) .rollrule_cache/ + +# local scheduler backups (same machine-specific paths as /scheduler/) +/scheduler.bak*/ +/scheduler_backup*/ diff --git a/README.md b/README.md index 18b3dc5..fb57332 100644 --- a/README.md +++ b/README.md @@ -240,13 +240,27 @@ cotdata-update --reconcile # prune stale manifest entries (see below) `--check` reports per-domain row counts, newest data date, last write, and any entries lagging behind their peers (a partial-run signal): ``` -domain entries rows newest data last write (UTC) behind -prices 84 829,096 2026-07-14 2026-07-15T10:15:24Z 1d -cot_legacy 44 70,201 2026-07-07 2026-07-14T04:26:55Z 8d +domain entries rows newest data last write (UTC) behind +prices 98 954,524 2026-08-07 2026-08-08T15:40:20Z 14d ← RETIRED +cot_legacy 53 81,821 2026-08-11 2026-08-21T12:10:42Z 10d +cot_disagg 29 29,187 2026-08-11 2026-08-21T12:10:51Z 10d ... -✓ all entries current (none lag behind their domain's newest). + +⚠ prices, metadata: RETIRED — this store still holds the files, but cotdata stopped writing + them at ADR-0007. Their dates are frozen and will never advance again. + Daily bars and contract specs now live in the marketdata store ($MARKETDATA_STORE); + read them with `marketdata-update --check`. + +✓ every entry in the live domains was written by the latest producer pass. ``` +**A `RETIRED` domain is not a warning you can wave away.** ADR-0007 moved bars and contract +specs to marketdata; a store created before that move still holds `prices/` and `metadata/`, +and nothing advances them. They earn a label rather than silence because lag is measured +*within* a domain — a uniformly frozen tree is perfectly self-consistent, so it scores +`lagging: 0` and reads exactly like a domain that ran cleanly minutes ago. See +[SYNCING.md](docs/SYNCING.md#retiring-the-moved-domains-prices-and-metadata) for removing them. + ### `status.json` — new-data signal for downstream tools Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the data), so tools that trigger on fresh data poll one small structured file instead of scanning the store: @@ -255,14 +269,16 @@ Every producer run writes `$COTDATA_STORE/status.json` (atomically, beside the d { "generated_at": "2026-07-15T10:15:24Z", "schema_version": 2, - "newest_data": { "prices": "2026-07-14", "cot_legacy": "2026-07-07", "cot_disagg": "2026-07-07", "cot_tff": "2026-07-07", "cot_supplemental": "2026-07-07" }, - "domains": { "prices": { "newest_data": "2026-07-14", "last_write": "2026-07-15T10:15:24Z", "entries": 84, "rows": 829096, "lagging": 0 }, "...": {} }, - "last_run": { "kinds": ["prices"], "ok": ["ES", "..."], "symbols_failed": [], "rows": 1658000, "seconds": 88, "at": "2026-07-15T10:15:24Z" } + "newest_data": { "cot_legacy": "2026-08-11", "cot_disagg": "2026-08-11", "cot_tff": "2026-08-11", "cot_supplemental": "2026-08-11", "prices": "2026-08-07" }, + "domains": { "cot_legacy": { "newest_data": "2026-08-11", "last_write": "2026-08-21T12:10:42Z", "entries": 53, "rows": 81821, "lagging": 0, "retired": false }, + "prices": { "newest_data": "2026-08-07", "last_write": "2026-08-08T15:40:20Z", "entries": 98, "rows": 954524, "lagging": 0, "retired": true }, "...": {} }, + "last_run": { "kinds": ["cot_legacy", "cot_disagg"], "ok": ["001602", "..."], "symbols_failed": [], "rows": 111008, "seconds": 22, "at": "2026-08-21T12:10:58Z" } } ``` **Polling contract:** -- To detect **new data**, compare `newest_data.` (e.g. `newest_data.prices`, `newest_data.cot_legacy`) against your last-seen value. It advances **only when genuinely new daily data arrives** — a no-op run leaves it unchanged. +- To detect **new data**, compare `newest_data.` (e.g. `newest_data.cot_legacy`) against your last-seen value. It advances **only when genuinely new daily data arrives** — a no-op run leaves it unchanged. +- **Check `domains..retired` first.** A retired domain's date is frozen, so a poller keyed on it waits forever and never errors — it just quietly reports data that stopped moving. This bullet used to give `newest_data.prices` as its example, and on 2026-08-21 a consumer following it read a two-week-old date from a store whose bars had moved to marketdata. **Bars are not in this store; poll `marketdata-update --check` instead.** - To detect that **a run happened at all** (new data or not), use `generated_at`. - `last_run` carries the most recent run's outcome (which domains, per-symbol failures) for alerting. diff --git a/docs/SYNCING.md b/docs/SYNCING.md index 66c2572..90ce509 100644 --- a/docs/SYNCING.md +++ b/docs/SYNCING.md @@ -224,6 +224,43 @@ disk usage and a directory listing both look right, and only a read notices. Bot transports match exclusions by **name at any depth**, so there is no `/manifests/` prefix to make the rule safe. Two stores, two passes, two lists. +### Retiring the moved domains: `prices/` and `metadata/` + +ADR-0007 moved daily bars and contract specs to marketdata. On a store created before that +move the old directories are still on disk, still full, and **still mirrored** — the pass-1 +exclusion list covers `_cache`, `_raw` and `citpy`, not these. So every sync faithfully +carries a frozen tree to every replica, indefinitely. + +**Why that is worse than it sounds.** `cotdata-update --check` measures lag *within* a +domain: each entry against its own domain's newest write. A tree where nothing has been +written since the cutover is perfectly self-consistent, so it scores `lagging: 0` and sits +in the report beside genuinely-current COT domains looking exactly as healthy. `status.json` +carried the same claim to machine consumers through `newest_data.prices`. + +On 2026-08-21 that is precisely what happened: a consumer read `newest_data.prices`, got +`2026-08-07` from a store whose bars had moved two weeks earlier, and had no signal it was +reading an abandoned tree. The bar store was current the whole time and its replica was +current too — nothing was broken except which directory the reader was pointed at. + +`--check` now labels both domains `RETIRED` and names the store the data moved to, so the +report no longer implies health it cannot verify. The files themselves still need removing: + +1. **Repoint the consumers first.** Bars come from `$MARKETDATA_STORE` via + `marketdata-update --check`. Deleting before repointing turns a silently-wrong reader + into a hard-failing one, which is better but still an outage. +2. **Delete on the producer, then let the mirror propagate.** `robocopy /MIR` and + `rsync --delete` both remove what the source no longer has, so one producer-side + deletion clears every replica on the next sync. Deleting on a replica instead + accomplishes nothing: the next sync restores it from the producer. +3. **Do not add `prices` to the pass-1 `/XD` list as a substitute.** Excluded directories + are neither copied *nor deleted*, so an exclusion freezes the stale copy on every + replica permanently instead of clearing it — the opposite of what you want. Exclude it + only *after* the deletion has propagated, and only if you want belt and braces. + +`metadata/` is small enough that leaving it costs nothing; `prices/` is typically ~46 MB per +replica. Neither is recoverable from cotdata once deleted, but both are fully reproducible +from marketdata, which is where they now live. + ### `vintage/` is irreplaceable, so where it is WRITTEN matters The vintage tree (`vintage/raw/`, `observations/`, `revisions/`, `snapshots.json`) records diff --git a/docs/WINDOWS_SCHEDULING.md b/docs/WINDOWS_SCHEDULING.md index 796a4ac..a36bb2d 100644 --- a/docs/WINDOWS_SCHEDULING.md +++ b/docs/WINDOWS_SCHEDULING.md @@ -24,7 +24,7 @@ New to Python and cotdata on Windows? Start with the [Windows Setup Guide](WINDO ## Wrapper scripts -Create **two** wrapper scripts — they run *different* commands from *different* packages: `marketdata-update` for the bars, `cotdata-cot` for the COT. (`cotdata-cot` is an alias of `cotdata-update`; it used to be half of a scoped pair, and the other half went with the price producers.) +Create **three** wrapper scripts — they run *different* commands from *different* packages: `marketdata-update` for the futures bars, `marketdata-update` again for the equities bars (a separate task, for the reasons below), and `cotdata-cot` for the COT. (`cotdata-cot` is an alias of `cotdata-update`; it used to be half of a scoped pair, and the other half went with the price producers.) > **Ready-made templates:** copy [`docs/examples/windows/run-prices.cmd`](examples/windows/run-prices.cmd) and [`run-cot.cmd`](examples/windows/run-cot.cmd) out of the repo into your `` (e.g. `C:\Users\you\cotdata\scheduler\`) rather than retyping them — then just fill in the placeholders. Keep them outside the repo so a `git pull` never clobbers your edited paths. @@ -73,6 +73,16 @@ set COTDATA_STORE=REPLACE_WITH_STORE_PATH Using the full venv `\Scripts\marketdata-update.exe` / `\Scripts\cotdata-cot.exe` path (rather than relying on the command being on `PATH`) matters here: Task Scheduler runs with a different, often bare, environment than your interactive shell, so a bare command name that resolves fine in Command Prompt can fail to resolve under the scheduler. +`run-equities.cmd` — the **equities/ETF** bars, from Yahoo rather than Norgate. Copy +[`docs/examples/windows/run-equities.cmd`](examples/windows/run-equities.cmd). It is a +separate task from the futures bars rather than a step inside `run-prices.cmd`, and the +file's header block gives the three reasons in full. The short version: `run-prices.cmd` +exits at its first command once the futures half has captured, so anything chained behind +it is unreachable on the repeats; a single flaky Yahoo symbol fails the whole equities run +and would otherwise abort the futures sync; and Yahoo needs no finals gate, so it can run +at 17:30 and be clear of the 20:55 task. **It takes no `--require-final`** — marketdata +refuses that flag on the equities domain rather than ignoring it. + `run-vintage.cmd` — **optional**, the as-published (vintage) capture. Copy [`docs/examples/windows/run-vintage.cmd`](examples/windows/run-vintage.cmd); it runs `cotdata-vintage fetch` then `ingest --pending`, both exit-code guarded. Two things to know @@ -88,19 +98,27 @@ before enabling it: ## Creating the tasks -Create three tasks (plus an optional fourth if you enable vintage capture) — times are the **machine's local** time; convert from ET if it isn't on Eastern: +Create four tasks (plus an optional fifth if you enable vintage capture) — times are the **machine's local** time; convert from ET if it isn't on Eastern: + +> **The bars task is named `cotdata prices` for historical reasons.** It was created before ADR-0007 moved bar production to `marketdata`, and renaming a live task loses its run history, so the name stayed. It runs `marketdata-update`, not anything in cotdata. Every `schtasks` line below uses the real name so you can paste it; on a fresh box `marketdata bars` is the better name and everything here still applies unchanged. ```bat :: 1) Bars — fire at the Continuous Futures Final (~8:55pm ET), then repeat every 15 min :: for 5 h. --require-final makes each repeat a cheap no-op until Norgate has actually :: pulled the Finals. /RI and /DU are what make this a poll: without them the task gets :: exactly one attempt per night. See "Polling with a repeating trigger" below. -schtasks /Create /TN "marketdata bars" /TR "\run-prices.cmd" /SC DAILY /ST 20:55 /RI 15 /DU 0005:00 +schtasks /Create /TN "cotdata prices" /TR "\run-prices.cmd" /SC DAILY /ST 20:55 /RI 15 /DU 0005:00 :: 2) COT — daily morning catch-up for holiday-delayed releases and as a safety net schtasks /Create /TN "cotdata COT (catch-up)" /TR "\run-cot.cmd" /SC DAILY /ST 08:10 -:: 3) Vintage (OPTIONAL) — as-published capture, ~90 min after the 15:30 ET release. +:: 3) Equities bars — Yahoo has the session's daily bar shortly after the 16:00 ET +:: close, so this needs neither a finals gate nor a repetition. Weekdays only, and +:: 17:30 keeps it clear of the 20:55 futures task so the two never run their +:: replica syncs at the same time. Its retry lives INSIDE run-equities.cmd. +schtasks /Create /TN "marketdata equities" /TR "\run-equities.cmd" /SC WEEKLY /D MON,TUE,WED,THU,FRI /ST 17:30 + +:: 4) Vintage (OPTIONAL) — as-published capture, ~90 min after the 15:30 ET release. :: Daily is deliberate: almost every request 304s, so it is nearly free, and it :: tightens the observed release date from a 7-day bound to a 1-day one. schtasks /Create /TN "cotdata vintage" /TR "\run-vintage.cmd" /SC DAILY /ST 17:00 @@ -112,6 +130,20 @@ schtasks /Create /TN "cotdata vintage" /TR "\run-vintage.cmd" /SC DAILY /ST > ``` > Every `Action` should be a full path to an existing `.cmd`. Fix a stray placeholder in place with `schtasks /Change /TN "cotdata COT (catch-up)" /TR "C:\real\path\run-cot.cmd"`. +> [!CAUTION] +> **Do not repoint the `cotdata prices` task at a chain wrapper on the strength of +> crowdmon's scheduling page.** That page +> ([`crowdmon/docs/WINDOWS_SCHEDULING.md`](https://github.com/mspinola/crowdmon/blob/main/docs/WINDOWS_SCHEDULING.md)) +> instructs `schtasks /Change /TN "cotdata prices" /TR "...\run-nightly.cmd"` so a panel +> publish can be chained behind the bars. crowdmon was **deprecated on 2026-08-07** and the +> chain was never installed on any box: there is no `run-nightly.cmd` and no +> `run-publish.cmd` for it to call. Running that command would point this box's futures +> producer at a file that does not exist, and it fails quietly — the task reports whatever +> the missing wrapper returns and the store simply stops advancing. +> +> The page is still online because crowdmon's `DEPRECATED.md` §2 keeps every file for +> citation. Treat it as a record of a design, not as instructions. + > **Prices task — two settings you must check now**, before this task will work unattended. Open it in `taskschd.msc` → Properties: > 1. **General tab → "Run only when user is logged on"** (the default — keep it). The prices task talks to the Norgate Data Updater, which only exists in your interactive desktop session; "run whether user is logged in or not" runs where NDU is invisible and the run fails. See [Norgate Data Updater needs an interactive session](#norgate-data-updater-needs-an-interactive-session). > 2. **Conditions tab → uncheck "Start the task only if the computer is on AC power"** (checked by default) if this is ever on a laptop — otherwise runs are silently skipped on battery. See [Task doesn't fire at all](#task-doesnt-fire-at-all). @@ -150,7 +182,7 @@ A **repetition on the trigger** fires on schedule regardless of what the previou ```bat :: convert an existing task in place -schtasks /Change /TN "marketdata bars" /RI 15 /DU 0005:00 +schtasks /Change /TN "cotdata prices" /RI 15 /DU 0005:00 ``` That produces `PT15MPT5H` on the trigger — the same shape the Friday COT poller above already uses, and the reason that one has always worked while the bars task did not. @@ -161,10 +193,65 @@ That produces `PT15MPT5H [!WARNING] +> **A guard keyed on `manifests/cot.json` never fires.** Measured on the reference box on +> 2026-08-21, across a real no-op repeat: that file's MD5 **changed**, while `status.json`'s +> `newest_data` map stayed byte-identical. Every manifest entry is rewritten on every pass so +> it carries a current `updated_at` (see `status.py`, and the lag check that depends on it), +> so the file churns even when nothing was fetched. Such a guard is worse than no guard: it +> looks like a fix and silently syncs every time anyway. +> +> `%%~tF` in a `.cmd` is wrong for a second, independent reason — it has **one-minute** +> resolution, so a capture landing in the same minute as the "before" reading compares equal +> and skips the sync that should have run. + +`newest_data` is the right signal and the documented one: it *"advances ONLY when genuinely +new data arrives"* (see [status.json](../README.md#statusjson--new-data-signal-for-downstream-tools)). +Snapshot it before the fetch and after, and skip the syncs only when the two agree. + +**Do not guard the daily catch-up as well.** This is the trap on the other side, and it costs +you bars rather than time. Both sync scripts mirror **both** stores since ADR-0007, so the +08:10 COT run is what carries **bar** data to the replicas on a morning after the bars task +captured but its own sync failed. COT is weekly, so `newest_data` does not move on roughly +four days in five — guarding the catch-up would skip the sync on exactly the days that +safety net exists for, and the bars would sit on the producer with nothing saying so. Give +the guard to the **poller** only, e.g. by having the Friday task pass a `--poll` argument the +catch-up does not: + +```powershell +$a = New-ScheduledTaskAction -Execute '\run-cot.cmd' -Argument '--poll' +Set-ScheduledTask -TaskName 'cotdata COT (Fri release)' -Action $a +``` + +(Use PowerShell rather than `schtasks /Change /TR` here: `schtasks` takes the whole quoted +string as the executable path, so the argument ends up part of the filename.) + +**Make the guard fail toward syncing.** If `status.json` is missing or unparseable, sync +anyway rather than skipping. A redundant mirror is merely slow; a skipped one is invisible, +which is the failure mode this whole page exists to prevent. + ## Testing your tasks Test in three layers: fire the task, read the result, then confirm it actually wrote data. The third layer is the one that matters — for cotdata the exit code alone is not a reliable success signal (see below). @@ -174,7 +261,7 @@ Test in three layers: fire the task, read the result, then confirm it actually w Don't wait for the trigger — run it now: ```bat -schtasks /Run /TN "marketdata bars" +schtasks /Run /TN "cotdata prices" ``` (Or in `taskschd.msc`: right-click the task → **Run**.) Running the *task* rather than the `.cmd` by hand is the stronger test: it exercises the scheduler's own account, environment, and working directory, which is where scheduled runs usually differ from your interactive shell. @@ -182,7 +269,7 @@ schtasks /Run /TN "marketdata bars" ### 2. Read what happened ```bat -schtasks /Query /TN "marketdata bars" /V /FO LIST +schtasks /Query /TN "cotdata prices" /V /FO LIST ``` Check **Last Run Time** and **Last Result**. For per-run detail, enable history once (right-click the task or the library root → **Enable All Tasks History**) and read the task's **History** tab. @@ -202,9 +289,9 @@ Confirm the relevant `newest data` date advanced (and `last write (UTC)` is rece - **A daytime bars run usually only proves the wrapper resolves** — if last night captured, the store already holds the newest settled bar, so a daytime run finds nothing newer and defers. The exception is worth knowing: if last night's run *failed or never fired*, a daytime run finds Norgate ahead of the store and captures immediately, so a missed night self-heals at the next trigger rather than waiting for the evening. To exercise the write path on demand regardless, run `marketdata-update --bars --domain futures` by hand (no `--require-final`), or fire the task after ~8:55pm ET. - **Test the trigger itself** by moving it a couple of minutes out, watching it fire, then setting it back: ```bat - schtasks /Change /TN "marketdata bars" /ST 14:20 + schtasks /Change /TN "cotdata prices" /ST 14:20 :: watch it run, then restore - schtasks /Change /TN "marketdata bars" /ST 20:55 + schtasks /Change /TN "cotdata prices" /ST 20:55 ``` This catches the two silent killers below — a disabled task, or the default *"only if on AC power"* condition skipping runs on a laptop. - **Keep a permanent record** by having the wrapper redirect output to a log file (see [Diagnosing a silent failure](#diagnosing-a-silent-failure)). @@ -261,7 +348,7 @@ If a task's General tab has **"Run whether user is logged in or not"** checked, `marketdata-update` exits non-zero on a hard fetch error, but a **deferred** `--require-final` run (NDU hasn't pulled the Finals yet) also exits non-zero — that's by design, not a bug, and the [repeating trigger](#polling-with-a-repeating-trigger) is what turns those into a working poll loop. Don't "fix" this by making the wrapper swallow the exit code: the repeats would still fire, but you would lose the only per-run signal separating a defer from a capture, and every repeat would do a full metadata fetch and replica sync instead of a cheap gate check. -To confirm a run actually wrote data, check `status.json` in the store (`newest_data.prices` advancing) rather than trusting Task Scheduler's Last Run Result alone — see [Operations](../README.md#operations) in the README. +To confirm a run actually wrote data, check the **marketdata** store rather than trusting Task Scheduler's Last Run Result alone: `marketdata-update --check`, and confirm the futures `last_date` advanced. **Not** cotdata's `status.json` — this line used to say `newest_data.prices` there, which is the retired domain ADR-0007 left behind. It is frozen at the cutover date, so it can never confirm anything about a bar run, and watching it shows a stall that is not happening (or hides one that is). See [Operations](../README.md#operations) in the README. ### Task Scheduler can't find `marketdata-update` / `cotdata-cot` diff --git a/docs/examples/windows/run-equities.cmd b/docs/examples/windows/run-equities.cmd new file mode 100644 index 0000000..f1ac559 --- /dev/null +++ b/docs/examples/windows/run-equities.cmd @@ -0,0 +1,119 @@ +@echo off +REM marketdata EQUITIES bar update wrapper for Windows Task Scheduler. +REM Copy this file into your scheduler folder and overwrite the two markers below. +REM Do NOT put angle brackets in a .cmd file: cmd reads them as redirection and +REM the file fails with "The syntax of the command is incorrect" even on comment +REM lines, which is why these are plain-text markers you replace. +REM REPLACE_WITH_MARKETDATA_STORE_PATH = your BAR store e.g. C:\Users\you\marketdata_store +REM REPLACE_WITH_VENV_PATH = your venv e.g. C:\Users\you\code\marketdata\.venv +REM +REM WHY THIS IS A SEPARATE TASK AND NOT A STEP IN run-prices.cmd +REM ------------------------------------------------------------------------ +REM Three reasons, any one of which is sufficient. +REM +REM 1. run-prices.cmd's first command is the gated futures fetch, which defers +REM with exit 1 the moment the store already holds Norgate's newest settled +REM session -- and the wrapper carries that code straight out, by design. Under +REM the repeating trigger that task polls every 15 min for 5 h, so on every +REM repeat after the futures half has captured, the wrapper exits at line one. +REM Anything chained behind it is unreachable on those repeats: equities would +REM get exactly one attempt per night with no retry. +REM +REM 2. The two halves fail differently. `--bars --domain equities` returns ok only +REM when failed == 0 (providers/yfinance.py: `"ok": failed == 0`), so a single +REM flaky Yahoo symbol fails the whole run. Chained into run-prices.cmd that +REM transient would abort the sync and push, and the futures bars written that +REM night would sit on the producer and never reach the replicas. One vendor +REM hiccup should not strand the other vendor's good data. +REM +REM 3. Nothing here needs Norgate's finals. Yahoo has the session's daily bar +REM shortly after the 16:00 ET close, so this task runs at 17:30 ET and is done +REM -- retries included -- before run-prices.cmd starts. That separation is +REM deliberate: both wrappers end by calling sync-store.cmd (robocopy /MIR) and +REM push-to-server.cmd (rsync --delete) against the same replicas, and two of +REM those running concurrently is a race nobody wants to debug. +REM +REM WHY NO --require-final +REM ------------------------------------------------------------------------ +REM It is futures-only and marketdata REFUSES it here rather than ignoring it: +REM update.py exits 2 with "--require-final is futures-only. yfinance publishes no +REM settled-versus-interim distinction". There is no settled/interim flag to gate +REM on, so the protection comes from cadence instead of a gate. It does: the +REM yfinance provider fetches period="max" and store.write_bars replaces the whole +REM parquet, so every run restates the full history. If a run ever captures an +REM in-progress or later-corrected bar, the NEXT day's run overwrites it. That +REM self-healing is why this is DAILY and not weekly or monthly -- on a monthly +REM cadence a bad capture would sit in the store for a month, unmarked, because +REM the store keeps no per-bar record of whether a value was provisional. +REM +REM WHY NO --metadata +REM ------------------------------------------------------------------------ +REM --metadata fetches FUTURES contract specs from Norgate. It is unrelated to the +REM equities half, and run-prices.cmd already runs it nightly. +REM +REM WHY THE RETRY IS IN HERE AND NOT IN THE TASK +REM ------------------------------------------------------------------------ +REM Task Scheduler's "if the task fails, restart every N minutes" does NOT fire on +REM a non-zero exit code from the action. It covers the engine failing to LAUNCH +REM the action. A run whose action returns 1 is logged as event 102, "Task +REM Scheduler successfully finished", and no restart is scheduled -- measured on +REM the reference box, where the futures task deferred on four consecutive nights +REM (2026-08-12..15) and was launched exactly once on each. See +REM docs/WINDOWS_SCHEDULING.md, "Polling with a repeating trigger". So the retry +REM has to live where it can actually run: here. +REM +REM A repetition trigger -- the futures task's answer -- would be the wrong shape +REM here. Equities have no --require-final gate, so a repeat after a success would +REM re-fetch every symbol and re-run both replica syncs rather than deferring +REM cheaply the way the futures task does. +REM +REM powershell Start-Sleep, not `timeout /t`: timeout reads the console and fails +REM with "Input redirection is not supported" under a scheduled task, which has +REM none. `ping -n` is the other classic dodge; Start-Sleep just says what it does. +REM +REM `if errorlevel 1` tests >= 1 and needs no expansion, so it is safe here. +REM `|| exit /b %ERRORLEVEL%` would NOT be: cmd expands %ERRORLEVEL% when it parses +REM the line, which is BEFORE the command on that line has run, so it would return +REM the previous command's code. On its own line, after the command, it is correct. +setlocal +set "MARKETDATA_STORE=REPLACE_WITH_MARKETDATA_STORE_PATH" +set "MDEXE=REPLACE_WITH_VENV_PATH\Scripts\marketdata-update.exe" + +REM Unscoped: every equities symbol the registry carries. --symbols would freeze +REM the universe at whatever was in the store the day this file was written, and a +REM symbol added to registry.yaml would then never be fetched by the only task +REM that fetches equities. +set "ATTEMPTS=3" +set "ATTEMPT=0" + +:fetch +set /a ATTEMPT+=1 +"%MDEXE%" --bars --domain equities +if not errorlevel 1 goto :fetched +REM Capture the code BEFORE anything else can clear it. `if` does not disturb +REM ERRORLEVEL, so the test above is safe to run first. +set "RC=%ERRORLEVEL%" +if %ATTEMPT% GEQ %ATTEMPTS% ( + echo equities fetch failed after %ATTEMPT% attempts, last code %RC% -- not syncing + exit /b %RC% +) +echo equities fetch failed with code %RC%, retrying in 5 min ^(attempt %ATTEMPT% of %ATTEMPTS%^) +powershell -NoProfile -NonInteractive -Command "Start-Sleep -Seconds 300" +goto :fetch + +:fetched + +REM --------------------------------------------------------------------------- +REM Chained replica syncs, same discipline and same order as run-prices.cmd: the +REM local-network sync first, the remote push second, so the near replica is +REM current even on a day the server is unreachable. Both scripts mirror BOTH +REM stores (ADR-0007), so the COT and futures passes here are cheap no-op rescans. +REM +REM The guard above means these are reached only when every symbol wrote. A +REM partial equities fetch is not mirrored: the retry loop re-runs the whole +REM full-history fetch, and the sync goes with the run that finally succeeds. +call "REPLACE_WITH_SCHEDULER_DIR\sync-store.cmd" +if errorlevel 1 exit /b %ERRORLEVEL% + +call "REPLACE_WITH_SCHEDULER_DIR\push-to-server.cmd" +exit /b %ERRORLEVEL% diff --git a/src/cotdata/status.py b/src/cotdata/status.py index ce80f35..fba1fe1 100644 --- a/src/cotdata/status.py +++ b/src/cotdata/status.py @@ -20,6 +20,23 @@ def status_path(): # Domains shown by --check, in report order. _DOMAINS = ["prices", "metadata", "cot", "cot_legacy", "cot_disagg", "cot_tff", "cot_supplemental"] + +# Domains this package no longer writes. ADR-0007 moved bars and contract specs to +# marketdata, and `store._DOMAIN_HALF` keeps them declared so a pre-move store can +# still migrate — but nothing advances them any more. +# +# They stay VISIBLE here rather than being dropped from _DOMAINS, because a store +# written before the move still holds the files and hiding them would make an +# operator's `--check` disagree with their own disk. What they must not do is +# report as healthy: `lagging` is measured WITHIN a domain (each entry against its +# domain's newest write), so a uniformly frozen tree scores a perfect zero and +# reads exactly like a domain that ran cleanly minutes ago. +# +# That is not hypothetical. On 2026-08-21 a consumer read `newest_data.prices` +# from a store whose bars had moved two weeks earlier, got 2026-08-07 with +# `lagging: 0` beside genuinely-current COT domains, and had no signal that it was +# reading an abandoned tree. +_RETIRED_DOMAINS = {"prices", "metadata"} # An entry whose producer last touched it more than this many days behind its # domain's newest write probably failed while its peers succeeded (a partial run). _LAG_DAYS = 3 @@ -87,9 +104,13 @@ def summarize(manifest: dict, today: Optional[dt.date] = None, parsed = {n: _parse_write(e.get("updated_at")) for n, e in entries.items()} usable = [w for w in parsed.values() if w] newest_write = max(usable) if usable else None + # A retired domain has no producer pass to be behind, so the within-domain + # lag check cannot say anything true about it. Reporting [] here is not the + # same as reporting health: `retired` below is what the reader is meant to + # act on, and format_report/build_status_doc both surface it. lagging = [] for name, e in entries.items(): - if name in ignore_lag: + if domain in _RETIRED_DOMAINS or name in ignore_lag: continue w = parsed[name] if w is None: @@ -102,6 +123,7 @@ def summarize(manifest: dict, today: Optional[dt.date] = None, if behind > _LAG_DAYS: lagging.append((name, e.get("updated_at"), behind)) out["domains"][domain] = { + "retired": domain in _RETIRED_DOMAINS, "entries": len(entries), "rows": sum(int(e.get("n_rows") or 0) for e in entries.values()), "newest": newest.isoformat() if newest else None, @@ -127,12 +149,27 @@ def format_report(manifest: dict, root: str = "", today: Optional[dt.date] = Non L.append("store is empty — no data written yet.") return "\n".join(L) - L.append(f"{'domain':<12}{'entries':>8}{'rows':>13}{'newest data':>14}" + L.append(f"{'domain':<18}{'entries':>8}{'rows':>13}{'newest data':>14}" f"{'last write (UTC)':>22}{'behind':>8}") for domain, d in s["domains"].items(): behind = "—" if d["behind_today"] is None else f"{d['behind_today']}d" - L.append(f"{domain:<12}{d['entries']:>8}{d['rows']:>13,}" - f"{str(d['newest']):>14}{str(d['last_write']):>22}{behind:>8}") + L.append(f"{domain:<18}{d['entries']:>8}{d['rows']:>13,}" + f"{str(d['newest']):>14}{str(d['last_write']):>22}{behind:>8}" + f"{' ← RETIRED' if d['retired'] else ''}") + + retired = [n for n, d in s["domains"].items() if d["retired"]] + if retired: + L.append("") + L.append(f"⚠ {', '.join(retired)}: RETIRED — this store still holds the files, but " + f"cotdata stopped writing") + L.append(f" {' ' * len(', '.join(retired))} them at ADR-0007. Their dates are frozen " + f"and will never advance again.") + L.append(" Daily bars and contract specs now live in the marketdata store " + "($MARKETDATA_STORE);") + L.append(" read them with `marketdata-update --check`. Delete these directories once " + "no") + L.append(" consumer reads them — see docs/SYNCING.md, \"Retiring the moved " + "domains\".") for domain, d in s["domains"].items(): if d["lagging"]: @@ -146,7 +183,11 @@ def format_report(manifest: dict, root: str = "", today: Optional[dt.date] = Non L.append(f" … and {len(d['lagging']) - 15} more") if not any(d["lagging"] for d in s["domains"].values()): L.append("") - L.append("✓ every entry was written by the latest producer pass.") + # Scoped to the live domains on purpose. A retired domain has no + # producer pass, so letting the all-clear speak for it is how a frozen + # tree passes for a fresh one. + L.append("✓ every entry in the live domains was written by the " + "latest producer pass.") return "\n".join(L) @@ -164,7 +205,11 @@ def build_status_doc(manifest: dict, last_run: Optional[dict] = None, Contract for external pollers: * ``newest_data[]`` — the date of the newest daily data for that domain. Advances ONLY when genuinely new data arrives → key on this to - detect "there is new data" (e.g. compare prices vs your last-seen date). + detect "there is new data" (e.g. compare cot_legacy vs your last-seen + date). CHECK ``domains[]["retired"]`` FIRST: a retired domain's + date is frozen, so a poller keyed on it waits forever without ever + erroring. This example used to read ``prices``, which is exactly such a + domain -- see ``_RETIRED_DOMAINS``. Bars live in the marketdata store now. * ``generated_at`` — refreshed on every producer run (new data or not) → key on this only to detect "a run happened". * ``last_run`` — outcome of the most recent run (kinds, ok/failed counts). @@ -177,6 +222,7 @@ def build_status_doc(manifest: dict, last_run: Optional[dict] = None, "entries": d["entries"], "rows": d["rows"], "lagging": len(d["lagging"]), + "retired": d["retired"], } for name, d in s["domains"].items() } diff --git a/tests/test_status.py b/tests/test_status.py index 8fcc565..c3b4921 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -12,7 +12,11 @@ def _manifest(): "NQ_backadj": {"last_date": "2026-07-05", "n_rows": 50, "updated_at": "2026-07-06T10:00:00Z"}, }, "cot_legacy": { - "001602": {"last_date": "2026-07-07", "n_rows": 1488, "updated_at": "2026-07-14T04:00:00Z"}, + "001602": {"last_date": "2026-07-07", "n_rows": 1488, "updated_at": "2026-07-15T04:00:00Z"}, + # Skipped by the latest pass: 9 days behind its peers' write time. Lives + # in a LIVE domain on purpose -- `prices` is retired, and the lag check + # is deliberately silent there (see the retired-domain tests below). + "099999": {"last_date": "2026-07-01", "n_rows": 12, "updated_at": "2026-07-06T04:00:00Z"}, }, } @@ -30,10 +34,10 @@ def test_summarize_counts_and_newest(): def test_summarize_flags_lagging_entry(): s = status.summarize(_manifest(), today=dt.date(2026, 7, 15)) - lagging = s["domains"]["prices"]["lagging"] - # NQ_backadj was last WRITTEN 2026-07-06, 9 days behind the domain's newest + lagging = s["domains"]["cot_legacy"]["lagging"] + # 099999 was last WRITTEN 2026-07-06, 9 days behind the domain's newest # write (2026-07-15) — i.e. the producer skipped it while its peers succeeded. - assert [name for name, _, _ in lagging] == ["NQ_backadj"] + assert [name for name, _, _ in lagging] == ["099999"] assert lagging[0][2] == 9 @@ -100,7 +104,7 @@ def test_format_report_contains_domain_and_lag_warning(): out = status.format_report(_manifest(), root="/store", today=dt.date(2026, 7, 15)) assert "prices" in out and "829" not in out # our synthetic totals, not the real store assert "250" in out # prices row total - assert "NQ_backadj" in out # lag warning lists the stale entry + assert "099999" in out # lag warning lists the stale entry assert "schema_version 2" in out @@ -115,7 +119,9 @@ def test_build_status_doc_flat_map_and_domains(): assert doc["newest_data"]["prices"] == "2026-07-14" assert doc["newest_data"]["cot_legacy"] == "2026-07-07" assert doc["domains"]["prices"]["rows"] == 250 - assert doc["domains"]["prices"]["lagging"] == 1 # NQ is stale + # `prices` is retired: it still reports entries/rows/newest, but never lag. + assert doc["domains"]["prices"]["lagging"] == 0 + assert doc["domains"]["cot_legacy"]["lagging"] == 1 # 099999 is stale assert doc["schema_version"] == 2 assert "generated_at" in doc @@ -148,3 +154,61 @@ def test_run_summary_ok_and_failed(): assert "1,234 rows" in line assert "newest 2026-07-14" in line assert "✗ GC: boom" in line + + +# ── retired domains ─────────────────────────────────────────────────────── +# ADR-0007 moved bars and contract specs to marketdata. The files stay in a +# pre-move store, so --check keeps showing them — but it must not let them read +# as healthy. `lagging` is measured WITHIN a domain, so a uniformly frozen tree +# scores zero and looks exactly like a domain that ran cleanly minutes ago. +# On 2026-08-21 a consumer read newest_data.prices from such a store, got a +# 14-day-old date with lagging: 0, and had no signal it was reading a dead tree. + +def test_the_retired_domains_are_flagged_as_retired(): + s = status.summarize(_manifest(), today=dt.date(2026, 7, 15)) + assert s["domains"]["prices"]["retired"] is True + assert s["domains"]["cot_legacy"]["retired"] is False + + +def test_a_retired_domain_is_never_reported_as_lagging(): + """Not because it is healthy — because the within-domain check cannot say + anything true about a domain with no producer pass. NQ_backadj is 9 days behind + its peers and would be flagged in any live domain.""" + s = status.summarize(_manifest(), today=dt.date(2026, 7, 15)) + assert s["domains"]["prices"]["lagging"] == [] + + +def test_the_report_names_the_retired_domains_and_says_where_the_data_went(): + out = status.format_report(_manifest(), today=dt.date(2026, 7, 15)) + assert "RETIRED" in out + assert "marketdata" in out + + +def test_the_all_clear_does_not_speak_for_the_retired_domains(): + """A store whose live domains are all current still holds a frozen `prices` + tree. The all-clear has to say WHICH entries it is vouching for, or it vouches + for the dead one too -- which is how a 14-day-old tree reads as healthy.""" + m = _manifest() + del m["cot_legacy"]["099999"] # no lag left in any live domain + out = status.format_report(m, today=dt.date(2026, 7, 15)) + assert "every entry in the live domains" in out + assert "RETIRED" in out + + +def test_status_json_carries_retired_so_a_poller_can_see_it(): + """The machine-readable contract needs the flag too: a poller keyed on + newest_data[] for a frozen domain waits forever without ever erroring.""" + doc = status.build_status_doc(_manifest(), today=dt.date(2026, 7, 15)) + assert doc["domains"]["prices"]["retired"] is True + assert doc["domains"]["cot_legacy"]["retired"] is False + + +def test_a_live_domain_still_reports_lagging_entries(): + """Guard against the retired suppression leaking into live domains.""" + m = _manifest() + m["cot_legacy"]["999999"] = {"last_date": "2026-07-07", "n_rows": 10, + "updated_at": "2026-06-01T04:00:00Z"} + s = status.summarize(m, today=dt.date(2026, 7, 15)) + assert {n for n, _, _ in s["domains"]["cot_legacy"]["lagging"]} == {"999999", "099999"} + # ...while the retired domain stays silent even with a stale entry of its own. + assert s["domains"]["prices"]["lagging"] == []