Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/ml4t/data/managers/storage_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,8 @@ def update(
f"Stored metadata provider for '{key}' must be a string, "
f"got {type(existing_provider).__name__}"
)
if not provider and existing_provider not in (None, "", "auto"):
provider = existing_provider

# Get the date range from existing data
_, last_timestamp = timestamp_bounds(existing_df)
Expand Down
48 changes: 48 additions & 0 deletions tests/test_update_provider_reuse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""A recorded provider must control acquisition, not only rewritten metadata."""

import pytest
from polars.testing import assert_frame_equal

from ml4t.data import DataManager
from ml4t.data.providers.synthetic import SyntheticProvider
from ml4t.data.storage.config import StorageConfig
from ml4t.data.storage.flat import FlatStorage
from ml4t.data.storage.hive import HiveStorage


@pytest.fixture(params=[FlatStorage, HiveStorage])
def stored_provider(request, tmp_path):
storage = request.param(StorageConfig(base_path=tmp_path))
with SyntheticProvider() as provider:
frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-07", "daily")
storage.write(frame, "equities/daily/SPY", {"provider": "synthetic"})
return storage, frame


def test_real_incremental_fetch_reuses_the_recorded_adapter_without_a_route(stored_provider):
storage, frame = stored_provider
with DataManager(storage=storage) as manager:
key = manager.update("SPY", fill_gaps=False)
assert key == "equities/daily/SPY"
assert manager.get_metadata_for_key(key)["provider"] == "synthetic"
assert storage.read(key).collect().height > frame.height


def test_explicit_provider_still_overrides_the_recorded_adapter(stored_provider):
storage, frame = stored_provider
storage.write(frame, "equities/daily/SPY", {"provider": "no-such-recorded-provider"})
with DataManager(storage=storage) as manager:
key = manager.update("SPY", fill_gaps=False, provider="synthetic")
assert manager.get_metadata_for_key(key)["provider"] == "synthetic"


def test_invalid_explicit_adapter_does_not_fall_back_to_the_recorded_one(stored_provider):
storage, frame = stored_provider
before = storage.get_metadata("equities/daily/SPY")
with (
DataManager(storage=storage) as manager,
pytest.raises(ValueError, match="not.*available|not found|Unknown|unknown"),
):
manager.update("SPY", fill_gaps=False, provider="not-a-provider")
assert storage.get_metadata("equities/daily/SPY") == before
assert_frame_equal(storage.read("equities/daily/SPY").collect(), frame)
Loading