From a0b589ee64030ac3466bad61f4dd3d5160db0304 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 10:34:30 -0400 Subject: [PATCH 1/2] test: reproduce stored-provider acquisition failure --- tests/test_update_provider_reuse.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/test_update_provider_reuse.py diff --git a/tests/test_update_provider_reuse.py b/tests/test_update_provider_reuse.py new file mode 100644 index 0000000..7bdda92 --- /dev/null +++ b/tests/test_update_provider_reuse.py @@ -0,0 +1,48 @@ +"""A recorded provider must control acquisition, not only rewritten metadata.""" + +import pytest +from polars.testing import assert_frame_equal + +from ml4t.data import DataManager +from ml4t.data.providers.synthetic import SyntheticProvider +from ml4t.data.storage.config import StorageConfig +from ml4t.data.storage.flat import FlatStorage +from ml4t.data.storage.hive import HiveStorage + + +@pytest.fixture(params=[FlatStorage, HiveStorage]) +def stored_provider(request, tmp_path): + storage = request.param(StorageConfig(base_path=tmp_path)) + with SyntheticProvider() as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-07", "daily") + storage.write(frame, "equities/daily/SPY", {"provider": "synthetic"}) + return storage, frame + + +def test_real_incremental_fetch_reuses_the_recorded_adapter_without_a_route(stored_provider): + storage, frame = stored_provider + with DataManager(storage=storage) as manager: + key = manager.update("SPY", fill_gaps=False) + assert key == "equities/daily/SPY" + assert manager.get_metadata_for_key(key)["provider"] == "synthetic" + assert storage.read(key).collect().height > frame.height + + +def test_explicit_provider_still_overrides_the_recorded_adapter(stored_provider): + storage, frame = stored_provider + storage.write(frame, "equities/daily/SPY", {"provider": "no-such-recorded-provider"}) + with DataManager(storage=storage) as manager: + key = manager.update("SPY", fill_gaps=False, provider="synthetic") + assert manager.get_metadata_for_key(key)["provider"] == "synthetic" + + +def test_invalid_explicit_adapter_does_not_fall_back_to_the_recorded_one(stored_provider): + storage, frame = stored_provider + before = storage.get_metadata("equities/daily/SPY") + with ( + DataManager(storage=storage) as manager, + pytest.raises(ValueError, match="not.*available|not found|Unknown|unknown"), + ): + manager.update("SPY", fill_gaps=False, provider="not-a-provider") + assert storage.get_metadata("equities/daily/SPY") == before + assert_frame_equal(storage.read("equities/daily/SPY").collect(), frame) From 8ff2fa0e2bdd424c90e51828dd037101aa811278 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 10:35:01 -0400 Subject: [PATCH 2/2] fix: select the stored provider before history checks and fetching --- src/ml4t/data/managers/storage_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ml4t/data/managers/storage_manager.py b/src/ml4t/data/managers/storage_manager.py index 7eb9925..8217885 100644 --- a/src/ml4t/data/managers/storage_manager.py +++ b/src/ml4t/data/managers/storage_manager.py @@ -508,6 +508,8 @@ def update( f"Stored metadata provider for '{key}' must be a string, " f"got {type(existing_provider).__name__}" ) + if not provider and existing_provider not in (None, "", "auto"): + provider = existing_provider # Get the date range from existing data _, last_timestamp = timestamp_bounds(existing_df)