From 1bd81889bef900d57e8c463fb65432dacab13aa8 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 20 Aug 2026 23:38:37 -0400 Subject: [PATCH 1/2] Refuse an unregistered --symbols instead of reporting wrote=0 The three providers disagreed about a symbol the registry does not carry. databento refused it (KeyError, "not in the marketdata registry"). yfinance and norgate filtered it out of their target list and returned ok=True with wrote=0, which main() turns into exit 0. So `--bars --symbols VIX` against a registry that no longer carries VIX printed one line and reported a successful run. A wrapper testing `if not errorlevel 1` reads that as a good fetch and goes on to sync a store nothing was written to. That is what a symbol retired out of the registry hands to its consumers: not a failure at the point of retirement, but a silent no-op months later, at whichever box still asks for it, looking like a typo. This is the rule the file already applies to --require-final and --windowed-n1-stats -- "a modifier that cannot modify anything is refused, not ignored" -- extended to the one argument that was still being ignored. Refused via p.error, so it exits 2 like the other malformed-request guards rather than 1, and a fetch failure stays distinguishable from a request that could never have fetched anything. The check is in main() rather than in each provider on purpose. An unscoped --bars runs every domain, so an equities symbol resolves to nothing in norgate and a futures symbol resolves to nothing in yfinance; a per-provider refusal would fail a run the other half handled perfectly well. There is a test for exactly that. A partially satisfiable request is refused whole. `--symbols SPY NOTASYMBOL` fetches nothing, because three of four symbols arriving is not the request that was made and the caller should decide, not the producer. Co-Authored-By: Claude Opus 5 --- src/marketdata/update.py | 21 ++++++++++++++ tests/test_cli_symbols.py | 61 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tests/test_cli_symbols.py diff --git a/src/marketdata/update.py b/src/marketdata/update.py index 1028439..c3f7715 100644 --- a/src/marketdata/update.py +++ b/src/marketdata/update.py @@ -143,6 +143,27 @@ def main(argv=None) -> int: p.error(f"{flag} modifies the databento ingest. Pass it with " f"--ingest-databento.") + # Same rule once more, for --symbols. A symbol the registry does not carry + # cannot be fetched by anything, and the providers disagreed about what to do + # with one: databento refuses it (KeyError, "not in the marketdata registry"), + # while yfinance and norgate filtered it out and returned ok=True with wrote=0 + # -- exit 0, a wrapper's `if not errorlevel 1` reading it as a good run. So + # `--symbols VIX` against a registry that no longer carries VIX reported + # SUCCESS while fetching nothing, and a symbol retired out of the registry + # took its consumers' fetches down silently, months from whoever did it. The + # check belongs here rather than in each provider: an unscoped `--bars` runs + # every domain, so a futures symbol legitimately resolves to nothing in + # yfinance and vice versa, and a per-provider refusal would fail a run the + # other half handled fine. + if args.symbols: + from .registry import REGISTRY + unknown = [s for s in args.symbols if s not in REGISTRY] + if unknown: + p.error("not in the marketdata registry: " + ", ".join(unknown) + + ". Nothing would be fetched for these, so this is refused " + "rather than reported as a run that wrote 0 rows. Check the " + "spelling, or add them to registry.yaml.") + if args.final_cutoff: print(f"note: --final-cutoff {args.final_cutoff} is deprecated and ignored. " f"The finals gate is data-driven (a newer settled bar than the store " diff --git a/tests/test_cli_symbols.py b/tests/test_cli_symbols.py new file mode 100644 index 0000000..8802f7f --- /dev/null +++ b/tests/test_cli_symbols.py @@ -0,0 +1,61 @@ +"""`marketdata-update --symbols`: a symbol the registry does not carry must be +refused, not filtered out. + +The providers used to disagree. databento raised KeyError on an unregistered +symbol; yfinance and norgate dropped it from their target list and returned +ok=True with wrote=0, which `main` turns into exit 0. So `--symbols VIX` against a +registry that no longer carried VIX printed a line nobody reads and reported a +successful run, and a wrapper testing `if not errorlevel 1` went on to sync a +store nothing had been written to. That is the failure mode a retired symbol +hands to its consumers, months after the retirement, looking like a typo. +""" +import pytest + +from marketdata import update +from marketdata.providers import norgate as nprov, yfinance as yprov + +YF_OK = {"kind": "bars_yahoo", "ok": True, "wrote": 1, "failed": 0} +NG_OK = {"kind": "bars_futures_norgate", "ok": True, "wrote": 1, "failed": 0, + "errors": [], "rows": 10, "newest": "2026-08-20"} + + +def test_an_unregistered_symbol_is_refused(capsys): + with pytest.raises(SystemExit): + update.main(["--bars", "--symbols", "NOTASYMBOL"]) + err = capsys.readouterr().err + assert "not in the marketdata registry" in err + assert "NOTASYMBOL" in err + + +def test_refused_even_when_other_requested_symbols_are_valid(capsys): + """A partially satisfiable request is still not the request that was made. If + one of four symbols has been retired out of the registry, the run must say so + rather than quietly fetch three and exit 0.""" + with pytest.raises(SystemExit): + update.main(["--bars", "--symbols", "SPY", "NOTASYMBOL"]) + err = capsys.readouterr().err + assert "NOTASYMBOL" in err + assert "SPY" not in err # names what is wrong, not what is fine + + +def test_a_registered_symbol_is_not_refused(monkeypatch): + """The guard must not over-fire on the case it exists to protect.""" + monkeypatch.setattr(yprov, "update", lambda *a, **k: YF_OK) + monkeypatch.setattr(nprov, "update", lambda *a, **k: NG_OK) + assert update.main(["--bars", "--domain", "equities", "--symbols", "VIX"]) == 0 + + +def test_an_equities_symbol_does_not_fail_the_futures_half(monkeypatch): + """An unscoped --bars runs every domain, so an equities symbol resolves to + nothing in norgate and a futures symbol resolves to nothing in yfinance. That + is why the check lives in main() and not in each provider: a per-provider + refusal would fail a run the other half handled perfectly well.""" + monkeypatch.setattr(yprov, "update", lambda *a, **k: YF_OK) + monkeypatch.setattr(nprov, "update", lambda *a, **k: NG_OK) + assert update.main(["--bars", "--symbols", "VIX"]) == 0 + + +def test_no_symbols_means_no_check(monkeypatch): + monkeypatch.setattr(yprov, "update", lambda *a, **k: YF_OK) + monkeypatch.setattr(nprov, "update", lambda *a, **k: NG_OK) + assert update.main(["--bars", "--domain", "equities"]) == 0 From f35b41213348b25729757121277f368d4a109ca2 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Fri, 21 Aug 2026 06:54:41 -0400 Subject: [PATCH 2/2] Sort the new test's imports ruff I001. `python -m ruff check src tests` is the CI lint step and the import block was written as one combined `from ... import a as x, b as y` line. Co-Authored-By: Claude Opus 5 --- tests/test_cli_symbols.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_symbols.py b/tests/test_cli_symbols.py index 8802f7f..a0971f4 100644 --- a/tests/test_cli_symbols.py +++ b/tests/test_cli_symbols.py @@ -12,7 +12,8 @@ import pytest from marketdata import update -from marketdata.providers import norgate as nprov, yfinance as yprov +from marketdata.providers import norgate as nprov +from marketdata.providers import yfinance as yprov YF_OK = {"kind": "bars_yahoo", "ok": True, "wrote": 1, "failed": 0} NG_OK = {"kind": "bars_futures_norgate", "ok": True, "wrote": 1, "failed": 0,