From c425ac1c867bf9f1a22888386ed02587af688d8e Mon Sep 17 00:00:00 2001 From: micahpw <6476273+micahpw@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:05:04 -0600 Subject: [PATCH] feat: generic multi-file simulation combining via BaseSimulation.from_paths Adds a shared, extensible way to treat N partition files (weekly/monthly chunks submitted in parallel) as one logical simulation, so extension authors get it for free without reimplementing the combining logic themselves -- while formats that combine through a fundamentally different mechanism can still override it. - interfaces.py: BaseSimulation.from_paths, a non-abstract classmethod. Default: one cls(path, **kwargs) per file, combined via the new MultiFileSimulation (simulations/multi_file.py) using the existing combine_overlapping_frames primitive. - PlexosDuckDBSimulation.from_paths overrides trivially -- its PlexosDuckDBSource already combines multiple files via SQL ATTACH, no need to route through the per-file-object model. - SiennaSimulation.from_paths overrides to read the across-file merge direction from the already-existing per-model SiennaModelConfig.merge field (previously only used for within-file block dedup), instead of a single hardcoded default -- a UC/decision model and an emulation model can legitimately want different behavior at a partition seam. - generic_aggregator.SimulationAggregator: additive parser_kwargs param only, backward compatible, future-proofs it for parser classes that need more than a bare path. - simulations/utils.py: dedup_slices no longer silently drops a block when two partitions share an identical start timestamp (was keyed by a plain start-time dict); combine_overlapping_frames gains self-documenting "earlier_wins"/"later_wins" aliases for the confusing "right"/"left" naming (kept for backward compatibility). New tests pin gap handling, differing columns, and timezone-mismatch behavior. - server/ingest.py: fixes two real, previously-undetected bugs found while wiring this up -- the Sienna branch silently used only the first of multiple simulation_paths (now uses from_paths); the PLEXOS branch imported a module that has never existed (gat.simulations.plexos), so it has always raised ImportError (now uses the duckdb backend, which needs no separate system file). tests/handlers/test_sienna_multifile.py and tests/simulations/test_combine_frames_aggregator.py pass unchanged, proving the legacy SimulationAggregator/scenariohandlers path is untouched. Full suite: 257 passed, 57 skipped, 0 failed (excluding test_plexos_regression.py, pre-existing-broken on this machine due to a local fixture mismatch, unrelated to this change). --- src/gat/interfaces.py | 30 +++++++ src/gat/server/ingest.py | 26 +++--- src/gat/simulations/generic_aggregator.py | 10 ++- src/gat/simulations/multi_file.py | 73 +++++++++++++++ src/gat/simulations/plexos_duckdb.py | 9 ++ src/gat/simulations/sienna_v1.py | 42 +++++++++ src/gat/simulations/utils.py | 43 +++++---- tests/server/__init__.py | 0 tests/server/test_ingest_create_parser.py | 66 ++++++++++++++ .../test_combine_overlapping_frames.py | 53 +++++++++++ tests/simulations/test_dedup_slices.py | 15 ++++ .../test_generic_aggregator_parser_kwargs.py | 52 +++++++++++ .../simulations/test_multi_file_simulation.py | 90 +++++++++++++++++++ .../test_sienna_simulation_from_paths.py | 82 +++++++++++++++++ 14 files changed, 563 insertions(+), 28 deletions(-) create mode 100644 src/gat/simulations/multi_file.py create mode 100644 tests/server/__init__.py create mode 100644 tests/server/test_ingest_create_parser.py create mode 100644 tests/simulations/test_generic_aggregator_parser_kwargs.py create mode 100644 tests/simulations/test_multi_file_simulation.py create mode 100644 tests/simulations/test_sienna_simulation_from_paths.py diff --git a/src/gat/interfaces.py b/src/gat/interfaces.py index f099823..e03ef27 100644 --- a/src/gat/interfaces.py +++ b/src/gat/interfaces.py @@ -17,6 +17,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from pathlib import Path import pandas as pd @@ -145,3 +146,32 @@ def get_default_compositions(self) -> list[DatasetComposition]: ActivePowerVariable__RenewableDispatch, etc. """ return [] + + @classmethod + def from_paths( + cls, paths: str | Path | list[str | Path], **kwargs + ) -> "BaseSimulation": + """Construct from one or more partition files (e.g. weekly/monthly + chunks of one logical simulation, submitted in parallel) treated + as a single combined simulation. + + Default: instantiate ``cls(path, **kwargs)`` once per path and + combine their datasets via + ``gat.simulations.multi_file.MultiFileSimulation`` — free for any + subclass whose constructor takes a single file path, which covers + most extensions, without them needing to implement multi-file + support themselves. + + Override this when your format combines through a different + mechanism than "N single-file objects + pandas concat" — e.g. SQL + ATTACH across converted files (see + ``PlexosDuckDBSimulation.from_paths``). + """ + path_list = [paths] if isinstance(paths, (str, Path)) else list(paths) + + if len(path_list) == 1: + return cls(path_list[0], **kwargs) + + from .simulations.multi_file import MultiFileSimulation + + return MultiFileSimulation(cls, path_list, **kwargs) diff --git a/src/gat/server/ingest.py b/src/gat/server/ingest.py index 4e2b952..6e0a17d 100644 --- a/src/gat/server/ingest.py +++ b/src/gat/server/ingest.py @@ -251,16 +251,15 @@ def _create_parser( raise ValueError("sienna handler requires simulation_paths") system = SiennaSystem(system_path) - # SiennaSimulation takes a single path; use first file - sim_path = ( - simulation_paths[0] - if isinstance(simulation_paths, list) - else simulation_paths - ) # ``model`` becomes ``selected_model`` inside the parser, which # decides which group of HDF5 datasets get exposed. sienna_model = None if model in (None, "default") else model - simulation = SiennaSimulation(sim_path, simulation=sienna_model) + # from_paths handles both a single path and multiple partition + # files (combined into one logical simulation) -- previously this + # silently dropped every file after the first. + simulation = SiennaSimulation.from_paths( + simulation_paths, simulation=sienna_model + ) return system, simulation elif handler == "reeds": @@ -281,14 +280,19 @@ def _create_parser( return system, simulation elif handler == "plexos": - from gat.systems.plexos import PlexosSystem - from gat.simulations.plexos import PlexosSimulation + # gat.systems.plexos / gat.simulations.plexos (the legacy H5 + # backend's would-be v1 wrappers) never existed as modules -- this + # branch has always raised ImportError. Use the duckdb backend, + # which already reads native PLEXOS Solution.zip/.duckdb files + # directly and needs no separate system file. + from gat.systems.plexos_duckdb import PlexosDuckDBSystem + from gat.simulations.plexos_duckdb import PlexosDuckDBSimulation if not simulation_paths: raise ValueError("plexos handler requires simulation_paths") - system = PlexosSystem() # Plexos may not have a separate system file - simulation = PlexosSimulation(simulation_paths) + system = PlexosDuckDBSystem(simulation_paths) + simulation = PlexosDuckDBSimulation.from_paths(simulation_paths) return system, simulation else: diff --git a/src/gat/simulations/generic_aggregator.py b/src/gat/simulations/generic_aggregator.py index a0c4fa7..06fe750 100644 --- a/src/gat/simulations/generic_aggregator.py +++ b/src/gat/simulations/generic_aggregator.py @@ -74,6 +74,7 @@ def __init__( parser_class: Type[BaseSimulationParser], parallel: bool = True, max_workers: Optional[int] = None, + parser_kwargs: Optional[Dict[str, Any]] = None, ): """ Initialize the simulation aggregator. @@ -83,6 +84,10 @@ def __init__( parser_class: Parser class to instantiate (must inherit from BaseSimulationParser) parallel: Whether to load files in parallel (default: True) max_workers: Maximum number of parallel workers (default: CPU count) + parser_kwargs: Extra keyword arguments forwarded to + ``parser_class(file_path, **parser_kwargs)`` for every file. + For parser classes whose constructor needs more than a bare + path (default: none). Raises: ValueError: If no valid files provided or parser_class is invalid @@ -96,6 +101,7 @@ def __init__( self.parser_class = parser_class self.parallel = parallel self.max_workers = max_workers or mp.cpu_count() + self.parser_kwargs = parser_kwargs or {} # Validate inputs if not self.file_paths: @@ -144,7 +150,7 @@ def _initialize_parsers_sequential(self) -> List[BaseSimulationParser]: logger.debug( f"Loading file {i + 1}/{len(self.file_paths)}: {file_path.name}" ) - parser = self.parser_class(str(file_path)) # type: ignore + parser = self.parser_class(str(file_path), **self.parser_kwargs) # type: ignore parsers.append(parser) except Exception as e: logger.error(f"Failed to load {file_path}: {e}") @@ -199,7 +205,7 @@ def _load_single_file(self, file_path: str) -> BaseSimulationParser: Returns: Initialized parser instance """ - return self.parser_class(file_path) # type: ignore + return self.parser_class(file_path, **self.parser_kwargs) # type: ignore def _validate_parsers(self): """Validate that all parsers have compatible simulation models.""" diff --git a/src/gat/simulations/multi_file.py b/src/gat/simulations/multi_file.py new file mode 100644 index 0000000..8a82cec --- /dev/null +++ b/src/gat/simulations/multi_file.py @@ -0,0 +1,73 @@ +"""Default multi-file BaseSimulation implementation. + +This is what BaseSimulation.from_paths() falls back to when a format +doesn't override it: one inner-class instance per path, datasets combined +with gat.simulations.utils.combine_overlapping_frames. It's the "parse +each file into a Python object, then let GAT combine them" case -- the +common one, free for any BaseSimulation subclass whose constructor takes +a single file path. + +Formats that combine through a different mechanism (e.g. PlexosDuckDBSimulation, +which ATTACHes multiple files at the SQL layer) override from_paths +directly instead of going through this class. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Type + +import pandas as pd + +from ..interfaces import BaseSimulation +from .utils import combine_overlapping_frames + +if TYPE_CHECKING: + from ..categories import CategoryMap + from ..datasets import DatasetComposition, DatasetInfo + + +class MultiFileSimulation(BaseSimulation): + """Combines N single-file BaseSimulation instances into one. + + Args: + inner_cls: A BaseSimulation subclass whose constructor accepts a + single path (plus optional kwargs). + paths: The partition files to combine. + merge_strategy: Passed to combine_overlapping_frames for every + dataset -- "earlier_wins"/"later_wins" (or the legacy + "right"/"left" spellings). Default "earlier_wins". + **inner_kwargs: Forwarded to inner_cls(path, **inner_kwargs) for + every instance. + """ + + def __init__( + self, + inner_cls: Type[BaseSimulation], + paths: list[str | Path], + merge_strategy: str = "earlier_wins", + **inner_kwargs, + ): + if len(paths) < 1: + raise ValueError("MultiFileSimulation requires at least one path") + self._instances = [inner_cls(p, **inner_kwargs) for p in paths] + self._merge_strategy = merge_strategy + + def list_datasets(self) -> list["DatasetInfo"]: + """Union of every instance's datasets by name -- metadata comes + from whichever instance first reported a given name.""" + by_name: dict[str, "DatasetInfo"] = {} + for instance in self._instances: + for info in instance.list_datasets(): + by_name.setdefault(info.name, info) + return list(by_name.values()) + + def get_dataset(self, name: str) -> pd.DataFrame: + frames = [instance.get_dataset(name) for instance in self._instances] + return combine_overlapping_frames(frames, merge_strategy=self._merge_strategy) + + def get_default_category_maps(self) -> list["CategoryMap"]: + return self._instances[0].get_default_category_maps() + + def get_default_compositions(self) -> list["DatasetComposition"]: + return self._instances[0].get_default_compositions() diff --git a/src/gat/simulations/plexos_duckdb.py b/src/gat/simulations/plexos_duckdb.py index 7f178d0..e7ceb07 100644 --- a/src/gat/simulations/plexos_duckdb.py +++ b/src/gat/simulations/plexos_duckdb.py @@ -85,6 +85,15 @@ def __init__( len(self._get_raw_tables()), ) + @classmethod + def from_paths(cls, paths, **kwargs) -> "PlexosDuckDBSimulation": + """Override of BaseSimulation.from_paths: PlexosDuckDBSource already + combines multiple files by ATTACHing each as its own DuckDB schema + (see gat.datahelpers.plexos_duckdb), so there's no need to route + through MultiFileSimulation's per-file-object model — just pass + the paths straight through.""" + return cls(paths, **kwargs) + @property def source(self) -> PlexosDuckDBSource: """Access the underlying PlexosDuckDBSource for advanced use.""" diff --git a/src/gat/simulations/sienna_v1.py b/src/gat/simulations/sienna_v1.py index 1d62782..93de3bb 100644 --- a/src/gat/simulations/sienna_v1.py +++ b/src/gat/simulations/sienna_v1.py @@ -89,6 +89,48 @@ def __init__( len(self._get_raw_datasets()), ) + @classmethod + def from_paths( + cls, + paths: str | Path | list[str | Path], + simulation: str | None = None, + **kwargs, + ) -> "BaseSimulation": + """Override of BaseSimulation.from_paths: the across-file merge + direction comes from the selected model's own + ``SiennaModelConfig.merge`` (already exposed as + ``SiennaSimulationParser.merge_strategy``) rather than + ``MultiFileSimulation``'s generic default -- a UC/decision model + and an emulation model can legitimately want different behavior + at a partition seam, and this reuses the existing per-model + setting instead of picking one global default. + """ + path_list = [paths] if isinstance(paths, (str, Path)) else list(paths) + + if len(path_list) == 1: + return cls(path_list[0], simulation=simulation, **kwargs) + + from .sienna import SiennaSimulationParser + + # Every partition of one logical simulation shares the same + # model, so the first file's config is representative -- this is + # a metadata-only read (SiennaSimulationConfig.from_h5_file opens + # the file in a `with` block), not a full data parse. + probe = SiennaSimulationParser(str(path_list[0])) + if simulation is not None: + probe.selected_model = simulation + merge_strategy = probe.merge_strategy or "earlier_wins" + + from .multi_file import MultiFileSimulation + + return MultiFileSimulation( + cls, + path_list, + merge_strategy=merge_strategy, + simulation=simulation, + **kwargs, + ) + @property def parser(self) -> object: """Access the underlying SiennaSimulationParser for advanced use.""" diff --git a/src/gat/simulations/utils.py b/src/gat/simulations/utils.py index f86d0e9..475d154 100644 --- a/src/gat/simulations/utils.py +++ b/src/gat/simulations/utils.py @@ -31,7 +31,17 @@ import polars as pl -block_combination_strategy = Literal["left", "right"] +block_combination_strategy = Literal["left", "right", "earlier_wins", "later_wins"] + +# "left"/"right" describe which side gets truncated at the overlap, not +# which side wins -- confusing enough that multiple docstrings in this +# module apologize for it. "earlier_wins"/"later_wins" are the canonical, +# self-documenting names going forward; "left"/"right" stay accepted for +# backward compatibility with existing call sites. +_MERGE_STRATEGY_ALIASES = { + "earlier_wins": "right", + "later_wins": "left", +} def resolve_compositions( @@ -81,20 +91,20 @@ def dedup_slices( See SiennaSimulationParser._get_decision_data() for detailed strategy examples. """ - # Create mapping of block start time to (original_index, block) - block_mapping = {} - for i, block in enumerate(blocks): - start_time = block.min() - block_mapping[start_time] = (i, block) - - # Sort blocks by start time - sorted_start_times = sorted(block_mapping.keys()) + # (start_time, original_index, block) triples, sorted by start_time. + # A plain dict keyed by start_time would silently clobber one block + # whenever two blocks share an identical start timestamp; sort() is + # stable, so ties instead keep their original input order. + indexed_blocks = sorted( + ((block.min(), i, block) for i, block in enumerate(blocks)), + key=lambda t: t[0], + ) + sorted_start_times = [t[0] for t in indexed_blocks] # Create result list in original order result = [None] * len(blocks) - for j, start_time in enumerate(sorted_start_times): - original_idx, current_block = block_mapping[start_time] + for j, (start_time, original_idx, current_block) in enumerate(indexed_blocks): if ignore_previous: # LEFT strategy: Keep data from current time forward, remove overlap with next block @@ -130,8 +140,7 @@ def dedup_slices( # RIGHT strategy: Remove overlap with previous block, keep data forward if j > 0: # Not the first block - find overlap with previous block - prev_start_time = sorted_start_times[j - 1] - prev_original_idx, prev_block = block_mapping[prev_start_time] + _, _, prev_block = indexed_blocks[j - 1] prev_end_time = prev_block.max() # Find where previous block ends in current block @@ -186,8 +195,11 @@ def combine_overlapping_frames( Args: frames: DataFrames to combine, each with a sortable time index. - merge_strategy: "right" (earlier wins, default) or "left" (later - wins) — see the module docstring for the visual explanation. + merge_strategy: "right"/"earlier_wins" (default) or "left"/ + "later_wins" — "earlier_wins"/"later_wins" are the canonical + names (self-documenting); "left"/"right" remain accepted for + backward compatibility. See the module docstring for the + visual explanation of what gets truncated. Returns: A single combined, chronologically-sorted DataFrame. @@ -197,6 +209,7 @@ def combine_overlapping_frames( if len(frames) == 1: return frames[0] + merge_strategy = _MERGE_STRATEGY_ALIASES.get(merge_strategy, merge_strategy) sorted_frames = sorted(frames, key=lambda df: df.index.min()) ignore_previous = merge_strategy == "left" diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/server/test_ingest_create_parser.py b/tests/server/test_ingest_create_parser.py new file mode 100644 index 0000000..a22a078 --- /dev/null +++ b/tests/server/test_ingest_create_parser.py @@ -0,0 +1,66 @@ +"""Regression tests for gat.server.ingest._create_parser's Sienna and +PLEXOS branches. + +Sienna: previously did `simulation_paths[0]`, silently dropping every +file after the first for a multi-file (partitioned) scenario. PLEXOS: +previously imported `gat.simulations.plexos`, a module that doesn't +exist -- the branch has always raised ImportError, so no PLEXOS +simulation could ever be constructed via the server ingest path. + +Uses mocks throughout -- no real Sienna/PLEXOS files needed, since only +_create_parser's own routing logic is under test here. +""" + +from unittest.mock import MagicMock, patch + +from gat.server.ingest import _create_parser + + +@patch("gat.simulations.sienna_v1.SiennaSimulation.from_paths") +@patch("gat.systems.sienna.SiennaSystem") +def test_sienna_multi_file_forwards_every_path(mock_system_cls, mock_from_paths): + mock_from_paths.return_value = MagicMock() + paths = ["week1.h5", "week2.h5", "week3.h5"] + + _create_parser("sienna", system_path="system.json", simulation_paths=paths) + + mock_from_paths.assert_called_once_with(paths, simulation=None) + + +@patch("gat.simulations.sienna_v1.SiennaSimulation.from_paths") +@patch("gat.systems.sienna.SiennaSystem") +def test_sienna_single_file_still_works(mock_system_cls, mock_from_paths): + mock_from_paths.return_value = MagicMock() + + _create_parser("sienna", system_path="system.json", simulation_paths=["only.h5"]) + + mock_from_paths.assert_called_once_with(["only.h5"], simulation=None) + + +@patch("gat.simulations.sienna_v1.SiennaSimulation.from_paths") +@patch("gat.systems.sienna.SiennaSystem") +def test_sienna_model_forwarded_when_not_default(mock_system_cls, mock_from_paths): + mock_from_paths.return_value = MagicMock() + + _create_parser( + "sienna", + system_path="system.json", + simulation_paths=["a.h5"], + model="UC", + ) + + mock_from_paths.assert_called_once_with(["a.h5"], simulation="UC") + + +@patch("gat.simulations.plexos_duckdb.PlexosDuckDBSimulation.from_paths") +@patch("gat.systems.plexos_duckdb.PlexosDuckDBSystem") +def test_plexos_no_longer_raises_import_error(mock_system_cls, mock_from_paths): + mock_from_paths.return_value = MagicMock() + paths = ["a.zip", "b.zip"] + + system, simulation = _create_parser( + "plexos", system_path=None, simulation_paths=paths + ) + + mock_system_cls.assert_called_once_with(paths) + mock_from_paths.assert_called_once_with(paths) diff --git a/tests/simulations/test_combine_overlapping_frames.py b/tests/simulations/test_combine_overlapping_frames.py index 97c527d..bcf42ee 100644 --- a/tests/simulations/test_combine_overlapping_frames.py +++ b/tests/simulations/test_combine_overlapping_frames.py @@ -76,3 +76,56 @@ def test_three_frames_chained_overlap_right(): out = combine_overlapping_frames([a, b, c], merge_strategy="right") # Earlier frame wins at each seam: 01-03 from a, 01-05 from b. assert out["v"].tolist() == [1, 2, 3, 4, 98, 6, 7] + + +def test_earlier_wins_later_wins_aliases_match_right_left(): + """ "earlier_wins"/"later_wins" are the canonical names for "right"/ + "left" -- same result either way.""" + a = _ts_frame(["2020-01-01", "2020-01-02", "2020-01-03"], [1, 2, 3]) + b = _ts_frame(["2020-01-03", "2020-01-04", "2020-01-05"], [99, 4, 5]) + assert combine_overlapping_frames([a, b], merge_strategy="earlier_wins").equals( + combine_overlapping_frames([a, b], merge_strategy="right") + ) + assert combine_overlapping_frames([a, b], merge_strategy="later_wins").equals( + combine_overlapping_frames([a, b], merge_strategy="left") + ) + + +def test_gap_between_partitions_passes_through_untouched(): + """A real gap (not just non-overlapping/adjacent) between partitions + isn't synthesized or filled -- it just passes through in the index.""" + a = _ts_frame(["2020-01-01", "2020-01-02"], [1, 2]) + b = _ts_frame(["2020-01-10", "2020-01-11"], [10, 11]) + out = combine_overlapping_frames([a, b]) + assert out["v"].tolist() == [1, 2, 10, 11] + assert out.index.tolist() == [ + pd.Timestamp("2020-01-01"), + pd.Timestamp("2020-01-02"), + pd.Timestamp("2020-01-10"), + pd.Timestamp("2020-01-11"), + ] + + +def test_differing_columns_unions_with_nan_fill(): + """Frames with different column sets union (outer join) rather than + silently dropping a column or raising.""" + a = _ts_frame(["2020-01-01", "2020-01-02"], [1, 2]) + b = _ts_frame(["2020-01-03", "2020-01-04"], [3, 4]) + b["w"] = [30, 40] + out = combine_overlapping_frames([a, b]) + assert set(out.columns) == {"v", "w"} + assert out["v"].tolist() == [1, 2, 3, 4] + assert pd.isna(out.loc[pd.Timestamp("2020-01-01"), "w"]) + assert out.loc[pd.Timestamp("2020-01-03"), "w"] == 30 + + +def test_timezone_mismatch_raises_clearly(): + """Combining a tz-naive frame with a tz-aware one raises a clear error + rather than silently misbehaving -- the comparison during sorting + fails before the broad except-and-fallback in the try block below it + ever gets a chance to swallow it.""" + a = _ts_frame(["2020-01-01", "2020-01-02"], [1, 2]) + b_idx = pd.DatetimeIndex(["2020-01-03", "2020-01-04"], tz="UTC") + b = pd.DataFrame({"v": [3, 4]}, index=b_idx) + with pytest.raises(TypeError): + combine_overlapping_frames([a, b]) diff --git a/tests/simulations/test_dedup_slices.py b/tests/simulations/test_dedup_slices.py index 0b9197a..3b3b003 100644 --- a/tests/simulations/test_dedup_slices.py +++ b/tests/simulations/test_dedup_slices.py @@ -67,6 +67,21 @@ def test_three_blocks_unsorted_input_preserves_input_order(): ] +def test_two_blocks_same_start_time_neither_dropped(): + """Two blocks sharing an identical start timestamp used to silently + clobber one entry in a start_time-keyed dict (block_mapping), losing + that block's result entirely. Both must now appear, with ties broken + by original input order.""" + a = _series("2020-01-01", "2020-01-02", "2020-01-03") + b = _series("2020-01-01", "2020-01-04", "2020-01-05") + out_left = dedup_slices([a, b], ignore_previous=True) + out_right = dedup_slices([a, b], ignore_previous=False) + assert len(out_left) == 2 + assert None not in out_left + assert len(out_right) == 2 + assert None not in out_right + + def test_overlap_dedup_for_chained_blocks_left(): """Chained overlap: A→B→C each overlapping by 1 timestamp.""" a = _series("2020-01-01", "2020-01-02", "2020-01-03") diff --git a/tests/simulations/test_generic_aggregator_parser_kwargs.py b/tests/simulations/test_generic_aggregator_parser_kwargs.py new file mode 100644 index 0000000..f334a4f --- /dev/null +++ b/tests/simulations/test_generic_aggregator_parser_kwargs.py @@ -0,0 +1,52 @@ +"""Regression test for SimulationAggregator's additive parser_kwargs +constructor param -- future-proofs the aggregator for a parser class whose +constructor needs more than a bare path (e.g. a PLEXOS-shaped +force_convert kwarg), without changing behavior for existing callers that +never pass it. +""" + +from gat.simulations.base import BaseSimulationParser +from gat.simulations.generic_aggregator import SimulationAggregator + + +class _FakeParser(BaseSimulationParser): + def __init__(self, file_path, tag="default"): + super().__init__() + self.file_path = file_path + self.tag = tag + + @property + def simulation_models(self): + return ["UC"] + + def list_raw_datasets(self): + return {"load": "load"} + + def get_raw_dataset(self, key): + return None + + +def _touch(tmp_path, name): + p = tmp_path / name + p.write_text("") + return p + + +def test_parser_kwargs_forwarded_sequential(tmp_path): + files = [_touch(tmp_path, "a.h5"), _touch(tmp_path, "b.h5")] + agg = SimulationAggregator( + file_paths=files, + parser_class=_FakeParser, + parallel=False, + parser_kwargs={"tag": "custom"}, + ) + assert all(p.tag == "custom" for p in agg.parsers) + + +def test_parser_kwargs_default_empty_dict(tmp_path): + files = [_touch(tmp_path, "a.h5")] + agg = SimulationAggregator( + file_paths=files, parser_class=_FakeParser, parallel=False + ) + assert agg.parser_kwargs == {} + assert agg.parsers[0].tag == "default" diff --git a/tests/simulations/test_multi_file_simulation.py b/tests/simulations/test_multi_file_simulation.py new file mode 100644 index 0000000..113d22b --- /dev/null +++ b/tests/simulations/test_multi_file_simulation.py @@ -0,0 +1,90 @@ +"""Unit tests for BaseSimulation.from_paths and its default fallback, +MultiFileSimulation. + +Uses a minimal fake BaseSimulation rather than a real format's parser -- +only the default-orchestration mechanism itself is under test here. +""" + +import pandas as pd +import pytest + +from gat.datasets import DatasetInfo, DatasetKind +from gat.interfaces import BaseSimulation +from gat.simulations.multi_file import MultiFileSimulation + + +class _FakeSimulation(BaseSimulation): + """One instance per path; each path maps to a fixed hourly frame.""" + + _DATA = { + "a": pd.date_range("2030-01-01", periods=24, freq="h"), + "b": pd.date_range("2030-01-03", periods=24, freq="h"), + "c": pd.date_range("2030-01-05", periods=24, freq="h"), + } + + def __init__(self, path, scale=1): + self.path = path + self.scale = scale + + def list_datasets(self): + return [ + DatasetInfo( + name="load", + description="d", + kind=DatasetKind.RAW_SIMULATION, + entity_column="e", + ) + ] + + def get_dataset(self, name): + idx = self._DATA[self.path] + return pd.DataFrame({"v": [self.scale] * len(idx)}, index=idx) + + def get_default_category_maps(self): + return [f"category-map-from-{self.path}"] + + def get_default_compositions(self): + return [f"composition-from-{self.path}"] + + +def test_single_path_returns_bare_instance_not_wrapped(): + result = _FakeSimulation.from_paths("a") + assert isinstance(result, _FakeSimulation) + assert not isinstance(result, MultiFileSimulation) + + +def test_multi_path_returns_multi_file_simulation(): + result = _FakeSimulation.from_paths(["a", "b"]) + assert isinstance(result, MultiFileSimulation) + + +def test_kwargs_forwarded_to_every_instance(): + result = _FakeSimulation.from_paths(["a", "b"], scale=5) + combined = result.get_dataset("load") + assert (combined["v"] == 5).all() + + +def test_list_datasets_unions_by_name(): + result = _FakeSimulation.from_paths(["a", "b"]) + names = [d.name for d in result.list_datasets()] + assert names == ["load"] + + +def test_get_dataset_combines_via_combine_overlapping_frames(): + result = _FakeSimulation.from_paths(["a", "b", "c"]) + combined = result.get_dataset("load") + assert len(combined) == 72 + assert combined.index.min() == pd.Timestamp("2030-01-01") + assert combined.index.max() == pd.Timestamp("2030-01-05 23:00:00") + assert combined.index.is_monotonic_increasing + + +def test_category_maps_and_compositions_come_from_first_instance(): + result = _FakeSimulation.from_paths(["a", "b"]) + assert result.get_default_category_maps() == ["category-map-from-a"] + assert result.get_default_compositions() == ["composition-from-a"] + + +def test_empty_path_list_raises(): + with pytest.raises(ValueError): + MultiFileSimulation(_FakeSimulation, []) diff --git a/tests/simulations/test_sienna_simulation_from_paths.py b/tests/simulations/test_sienna_simulation_from_paths.py new file mode 100644 index 0000000..8b05e7c --- /dev/null +++ b/tests/simulations/test_sienna_simulation_from_paths.py @@ -0,0 +1,82 @@ +"""Unit tests for SiennaSimulation.from_paths -- specifically that the +across-file merge direction comes from the selected model's own +SiennaModelConfig.merge (via SiennaSimulationParser.merge_strategy) +rather than a hardcoded default, since a UC/decision model and an +emulation model can legitimately want different behavior at a partition +seam. + +Uses a mocked SiennaSimulationParser rather than real fixture data -- +only the from_paths routing/merge-strategy-selection logic is under test +here; tests/handlers/test_sienna_multifile.py covers the real, +Docker-fixture-based integration path. +""" + +from unittest.mock import MagicMock, patch + +from gat.simulations.multi_file import MultiFileSimulation +from gat.simulations.sienna_v1 import SiennaSimulation + + +def _mock_simulation(monkeypatch, merge_strategy): + """Patch SiennaSimulation's dependencies so construction never touches + a real file, and the underlying parser reports the given merge + strategy.""" + + def fake_init(self, simulation_path, simulation=None, compositions=None): + self._parser = MagicMock() + self._parser.simulation = "Emulator" + self._parser.merge_strategy = merge_strategy + self._compositions = {} + self._raw_datasets = {} + self._resolved_compositions = {} + + monkeypatch.setattr(SiennaSimulation, "__init__", fake_init) + + +@patch("gat.simulations.sienna.SiennaSimulationParser") +def test_single_path_bypasses_multi_file_simulation(mock_parser_cls, monkeypatch): + _mock_simulation(monkeypatch, merge_strategy="right") + result = SiennaSimulation.from_paths("a.h5") + assert isinstance(result, SiennaSimulation) + assert not isinstance(result, MultiFileSimulation) + mock_parser_cls.assert_not_called() # no probe needed for a single path + + +@patch("gat.simulations.sienna.SiennaSimulationParser") +def test_multi_path_uses_probed_merge_strategy(mock_parser_cls, monkeypatch): + probe = MagicMock() + probe.merge_strategy = "left" + mock_parser_cls.return_value = probe + + _mock_simulation(monkeypatch, merge_strategy="left") + result = SiennaSimulation.from_paths(["a.h5", "b.h5"]) + + assert isinstance(result, MultiFileSimulation) + assert result._merge_strategy == "left" + mock_parser_cls.assert_called_once_with("a.h5") + + +@patch("gat.simulations.sienna.SiennaSimulationParser") +def test_multi_path_sets_selected_model_on_probe_when_given( + mock_parser_cls, monkeypatch +): + probe = MagicMock() + probe.merge_strategy = "right" + mock_parser_cls.return_value = probe + + _mock_simulation(monkeypatch, merge_strategy="right") + SiennaSimulation.from_paths(["a.h5", "b.h5"], simulation="UC") + + assert probe.selected_model == "UC" + + +@patch("gat.simulations.sienna.SiennaSimulationParser") +def test_multi_path_falls_back_when_probe_reports_none(mock_parser_cls, monkeypatch): + probe = MagicMock() + probe.merge_strategy = None + mock_parser_cls.return_value = probe + + _mock_simulation(monkeypatch, merge_strategy=None) + result = SiennaSimulation.from_paths(["a.h5", "b.h5"]) + + assert result._merge_strategy == "earlier_wins"