From 654fb140e69e6a46ea32ca44a88abae29bc2d825 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 00:00:09 +0200 Subject: [PATCH 01/61] docs(examples): add a Coello distributed run driven from one NetCDF Standalone script replaying the workflow behind test_e2e_coello_from_netcdf.py::TestMuskingumPipeline:: test_the_drivers_come_from_the_file_and_cover_the_model: one MeteoInputs.from_netcdf call replaces the three raster-folder reads, through the routed fields, per-gauge metrics and the saved rasters. --- .../coello-distributed-model-run-netcdf.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py new file mode 100644 index 00000000..870c1a42 --- /dev/null +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py @@ -0,0 +1,111 @@ +"""Distributed Muskingum model driven from a single combined NetCDF. + +Standalone version of the workflow behind +`tests/rrm/catchment/test_e2e_coello_from_netcdf.py::TestMuskingumPipeline:: +test_the_drivers_come_from_the_file_and_cover_the_model`: one `MeteoInputs.from_netcdf` call +replaces the three raster-folder reads, so the model touches no meteorological raster at all. +`meteo.nc` packs the rainfall, temperature and evapotranspiration folders bundled under +`tests/rrm/data/coello/{prec,temp,evap}` into one file with the calendar inside it -- see +`tests/rrm/data/coello/convert_and_combine_meteo_inputs_to_netcdf.py` for how it was built. +""" + +from __future__ import annotations + +import numpy as np + +from hapi.catchment import Catchment +from hapi.inputs import FlowNetwork, MeteoInputs +from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 as HBV +from hapi.run import Run + +# %% Paths +Path = "tests/rrm/data/coello" +MeteoPath = f"{Path}/meteo.nc" +FlowAccPath = f"{Path}/gis/acc4000.tif" +FlowDPath = f"{Path}/gis/fd4000.tif" +ParPath = f"{Path}/parameters/muskingum" +GaugesTablePath = f"{Path}/calibration/gauges.csv" +GaugesPath = f"{Path}/calibration" + +# %% Meteorological data -- the whole point: one file, no raster reads +AreaCoeff = 1530 +InitialCond = [0, 5, 5, 5, 0] +Snow = 0 + +start = "2009-01-01" +end = "2009-01-10" +name = "Coello" + +Coello = Catchment(name, start, end, spatial_resolution="Distributed") +Coello.meteo = MeteoInputs.from_netcdf( + MeteoPath, + precipitation="precipitation", + temperature="temperature", + evapotranspiration="evapotranspiration", +) + +Coello.flow_network = FlowNetwork.from_rasters(FlowAccPath, FlowDPath) +Coello.read_parameters(ParPath, Snow, maxbas=False) +Coello.read_lumped_model(HBV, AreaCoeff, InitialCond) + +# %% Gauges +Coello.read_gauge_table(GaugesTablePath, FlowAccPath) +Coello.read_discharge_gauges(GaugesPath, column="id", fmt="%Y-%m-%d") + +# %% Check the drivers actually came from the file and cover the model +print(f"meteo grid + steps : {Coello.meteo.shape}") +print(f"model steps : {len(Coello.date_index)}") +print(f"meteo period : {Coello.meteo.time[0]} -> {Coello.meteo.time[-1]}") +print(f"model period : {Coello.date_index[0]} -> {Coello.date_index[-1]}") +assert Coello.meteo.time_steps == len(Coello.date_index), ( + "the drivers must hold exactly as many steps as the model spans" +) +assert Coello.meteo.time[0] == Coello.date_index[0], "the drivers must start where the model does" +assert Coello.meteo.time[-1] == Coello.date_index[-1], "the drivers must end where the model does" + +# %% Run the model +""" +Outputs: + ---------- + 1-state_variables: [numpy attribute] + 4D array (rows,cols,time,states) states are [sp,wc,sm,uz,lv] + 2-qlz: [numpy attribute] + 3D array of the lower zone discharge + 3-quz: [numpy attribute] + 3D array of the upper zone discharge + 4-qout: [numpy attribute] + 1D timeseries of discharge at the outlet of the catchment + of unit m3/sec + 5-quz_routed: [numpy attribute] + 3D array of the upper zone discharge accumulated and + routed at each time step + 6-qlz_translated: [numpy attribute] + 3D array of the lower zone discharge translated at each time step +""" +Run.RunHapi(Coello) + +# %% Routed fields cover the grid, finite inside the catchment +inside = ~np.isnan(Coello.flow_network.flow_acc_arr) +for field_name in ("Qtot", "quz_routed", "qlz_translated"): + field = getattr(Coello, field_name) + print(f"{field_name:15s} shape {field.shape}, finite inside: {np.isfinite(field[inside]).all()}") + +# %% Extract discharge at every gauge and score against the observations +Coello.extract_discharge(calculate_metrics=True) + +for gauge_id in Coello.GaugesTable["id"]: + print("----------------------------------") + print(f"Gauge - {gauge_id}") + print(f"RMSE= {Coello.metrics.loc['RMSE', gauge_id]:.2f}") + print(f"NSE= {Coello.metrics.loc['NSE', gauge_id]:.2f}") + print(f"NSEhf= {Coello.metrics.loc['NSEhf', gauge_id]:.2f}") + print(f"KGE= {Coello.metrics.loc['KGE', gauge_id]:.2f}") + print(f"WB= {Coello.metrics.loc['WB', gauge_id]:.2f}") + +# %% Save the routed discharge to rasters, one per time step +SaveTo = "results/saved rasters/" +Coello.save_results(flow_acc_path=FlowAccPath, result=1, path=SaveTo) +print(f"rasters written to : {SaveTo}") + +# %% Plot the hydrograph at the outlet gauge (row position, not the gauge id) +Coello.plot_hydrograph(start, end, Coello.GaugesTable.index[-1]) From 6f8ca71303edd0cca092aa2b3ebc4a191bc53f24 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 01:18:23 +0200 Subject: [PATCH 02/61] feat(config): build a Catchment from a YAML run configuration Every example run script opened with a block of hardcoded path and constant assignments, then called the `read_*` methods in the exact order the build-then-mutate pattern requires. Configuring a case study meant editing Python, and the ordering constraint was rediscovered per script. `hapi.config` lifts that block into a YAML file: `load_config` parses it into a `RunConfig` of dataclasses, and `from_yaml` reads every input and assigns it onto the model in the required order. `Catchment.from_yaml` exposes the same thing as an alternate constructor, delegating through a local import because `hapi.config` imports `Catchment` to build one. Running the model stays the caller's job, as in a hand-wired script. The schema covers lumped and distributed runs, which disagree on the shape of two blocks: `meteo` is a grid for distributed and a single CSV for lumped, and `gauges.discharge` is a folder of per-gauge CSVs against a single file. A name-to-class registry resolves `conceptual_model.model_class`, since the model is a class rather than data. `catchment.routing_method` is canonicalised to the exact literal the internals compare against: `Catchment.__init__` stores it verbatim, and `distrrm.SpatialRouting` tests `!= "Muskingum"` case-sensitively, so a lowercase spelling silently sent every cell down the MAXBAS branch and read `bankfull_depth`, which is None outside the flood model. Also fix three methods whose `str | dt.datetime` parameters called `strptime` unconditionally, so passing the documented `datetime` raised TypeError: `plot_hydrograph`, `read_discharge_gauges` (the `split` path) and `save_results`. Each now branches on `isinstance(..., str)`. - add `pyyaml` as a dependency; it was only present transitively - port coello-distributed-model-run-netcdf.py onto `Catchment.from_yaml` --- .../coello-distributed-model-run-netcdf.py | 60 +-- .../coello-distributed-model-run-netcdf.yaml | 37 ++ pixi.lock | 1 + pyproject.toml | 3 +- src/hapi/catchment.py | 34 +- src/hapi/config.py | 417 ++++++++++++++++++ 6 files changed, 506 insertions(+), 46 deletions(-) create mode 100644 examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml create mode 100644 src/hapi/config.py diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py index 870c1a42..40d98a7d 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py @@ -1,4 +1,4 @@ -"""Distributed Muskingum model driven from a single combined NetCDF. +"""Distributed Muskingum model driven from a single combined NetCDF, built from YAML. Standalone version of the workflow behind `tests/rrm/catchment/test_e2e_coello_from_netcdf.py::TestMuskingumPipeline:: @@ -7,6 +7,10 @@ `meteo.nc` packs the rainfall, temperature and evapotranspiration folders bundled under `tests/rrm/data/coello/{prec,temp,evap}` into one file with the calendar inside it -- see `tests/rrm/data/coello/convert_and_combine_meteo_inputs_to_netcdf.py` for how it was built. + +Everything that used to be a "Paths" block of hardcoded assignments now lives in +`coello-distributed-model-run-netcdf.yaml`, next to this script -- `Catchment.from_yaml` reads +it and assembles the `Catchment` the same way `_build` did in the e2e test. """ from __future__ import annotations @@ -14,44 +18,13 @@ import numpy as np from hapi.catchment import Catchment -from hapi.inputs import FlowNetwork, MeteoInputs -from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 as HBV from hapi.run import Run -# %% Paths -Path = "tests/rrm/data/coello" -MeteoPath = f"{Path}/meteo.nc" -FlowAccPath = f"{Path}/gis/acc4000.tif" -FlowDPath = f"{Path}/gis/fd4000.tif" -ParPath = f"{Path}/parameters/muskingum" -GaugesTablePath = f"{Path}/calibration/gauges.csv" -GaugesPath = f"{Path}/calibration" - -# %% Meteorological data -- the whole point: one file, no raster reads -AreaCoeff = 1530 -InitialCond = [0, 5, 5, 5, 0] -Snow = 0 - -start = "2009-01-01" -end = "2009-01-10" -name = "Coello" - -Coello = Catchment(name, start, end, spatial_resolution="Distributed") -Coello.meteo = MeteoInputs.from_netcdf( - MeteoPath, - precipitation="precipitation", - temperature="temperature", - evapotranspiration="evapotranspiration", +# %% Load the configuration and build the model +Coello = Catchment.from_yaml( + "examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml" ) -Coello.flow_network = FlowNetwork.from_rasters(FlowAccPath, FlowDPath) -Coello.read_parameters(ParPath, Snow, maxbas=False) -Coello.read_lumped_model(HBV, AreaCoeff, InitialCond) - -# %% Gauges -Coello.read_gauge_table(GaugesTablePath, FlowAccPath) -Coello.read_discharge_gauges(GaugesPath, column="id", fmt="%Y-%m-%d") - # %% Check the drivers actually came from the file and cover the model print(f"meteo grid + steps : {Coello.meteo.shape}") print(f"model steps : {len(Coello.date_index)}") @@ -60,8 +33,12 @@ assert Coello.meteo.time_steps == len(Coello.date_index), ( "the drivers must hold exactly as many steps as the model spans" ) -assert Coello.meteo.time[0] == Coello.date_index[0], "the drivers must start where the model does" -assert Coello.meteo.time[-1] == Coello.date_index[-1], "the drivers must end where the model does" +assert Coello.meteo.time[0] == Coello.date_index[0], ( + "the drivers must start where the model does" +) +assert Coello.meteo.time[-1] == Coello.date_index[-1], ( + "the drivers must end where the model does" +) # %% Run the model """ @@ -88,7 +65,9 @@ inside = ~np.isnan(Coello.flow_network.flow_acc_arr) for field_name in ("Qtot", "quz_routed", "qlz_translated"): field = getattr(Coello, field_name) - print(f"{field_name:15s} shape {field.shape}, finite inside: {np.isfinite(field[inside]).all()}") + print( + f"{field_name:15s} shape {field.shape}, finite inside: {np.isfinite(field[inside]).all()}" + ) # %% Extract discharge at every gauge and score against the observations Coello.extract_discharge(calculate_metrics=True) @@ -103,9 +82,12 @@ print(f"WB= {Coello.metrics.loc['WB', gauge_id]:.2f}") # %% Save the routed discharge to rasters, one per time step +# save_results re-reads the flow-accumulation raster for georeferencing; FlowNetwork keeps only +# the arrays, not the source path, so this repeats the path already given in the YAML. +FlowAccPath = "tests/rrm/data/coello/gis/acc4000.tif" SaveTo = "results/saved rasters/" Coello.save_results(flow_acc_path=FlowAccPath, result=1, path=SaveTo) print(f"rasters written to : {SaveTo}") # %% Plot the hydrograph at the outlet gauge (row position, not the gauge id) -Coello.plot_hydrograph(start, end, Coello.GaugesTable.index[-1]) +Coello.plot_hydrograph(Coello.start, Coello.end, Coello.GaugesTable.index[-1]) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml new file mode 100644 index 00000000..29fb5a8b --- /dev/null +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml @@ -0,0 +1,37 @@ +catchment: + name: Coello + start: "2009-01-01" + end: "2009-01-10" + spatial_resolution: distributed + temporal_resolution: daily + routing_method: muskingum + +meteo: + source: netcdf + path: tests/rrm/data/coello/meteo.nc + precipitation: precipitation + temperature: temperature + evapotranspiration: evapotranspiration + +flow_network: + flow_accumulation: tests/rrm/data/coello/gis/acc4000.tif + flow_direction: tests/rrm/data/coello/gis/fd4000.tif + +parameters: + path: tests/rrm/data/coello/parameters/muskingum + snow: false + maxbas: false + +conceptual_model: + model_class: HBVBergestrom92 + catchment_area: 1530 + initial_condition: [0, 5, 5, 5, 0] + +gauges: + table: tests/rrm/data/coello/calibration/gauges.csv + discharge: tests/rrm/data/coello/calibration + column: id + fmt: "%Y-%m-%d" + +outputs: + results_dir: "results/saved rasters/" diff --git a/pixi.lock b/pixi.lock index cd77a542..028b126c 100644 --- a/pixi.lock +++ b/pixi.lock @@ -3374,6 +3374,7 @@ packages: - oasis-optimization>=1.0.3 - cleopatra>=0.32.0 - matplotlib>=3.11.0 + - pyyaml>=6.0 - earthlens[ecmwf]>=0.12.0 ; extra == 'inputs' requires_python: '>=3.11,<4' - pypi: https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl diff --git a/pyproject.toml b/pyproject.toml index b6a2163c..aaeff9e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ dependencies = [ "statista >=0.8.0", "Oasis-Optimization >=1.0.3", "cleopatra >=0.32.0", - "matplotlib >=3.11.0" + "matplotlib >=3.11.0", + "pyyaml >=6.0" ] [project.optional-dependencies] diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 2ed84afc..132df5a8 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -190,6 +190,24 @@ def __init__( self.Qsim: np.ndarray | None = None self.metrics: pd.DataFrame | None = None + @classmethod + def from_yaml(cls, path: str) -> Catchment: + """Read a YAML run configuration and assemble a `Catchment` from it. + + Delegates to `hapi.config.from_yaml`, imported inside this method rather than at + module level: `hapi.config` imports `Catchment` itself (to build one), so a top-level + import here would be circular. + + Args: + path: Path to the YAML file. See `hapi.config` for the schema. + + Returns: + Catchment: The model, with every input read, parsed and assigned. + """ + from hapi.config import from_yaml + + return from_yaml(path) + def read_flow_path_length(self, path: str): """Read the flow path length raster. @@ -674,8 +692,10 @@ def read_discharge_gauges( self.QGauges[f.columns[0]] = f.loc[self.start : self.end, f.columns[0]] if split: - start_date = dt.datetime.strptime(start_date, fmt) - end_date = dt.datetime.strptime(end_date, fmt) + if isinstance(start_date, str): + start_date = dt.datetime.strptime(start_date, fmt) + if isinstance(end_date, str): + end_date = dt.datetime.strptime(end_date, fmt) self.QGauges = self.QGauges.loc[start_date:end_date] logger.debug("Gauges data are read successfully") @@ -890,8 +910,10 @@ def plot_hydrograph( tuple: A tuple of (fig, ax) where fig is the matplotlib Figure and ax is the matplotlib Axes object. """ - start_date = dt.datetime.strptime(start_date, fmt) - end_date = dt.datetime.strptime(end_date, fmt) + if isinstance(start_date, str): + start_date = dt.datetime.strptime(start_date, fmt) + if isinstance(end_date, str): + end_date = dt.datetime.strptime(end_date, fmt) fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 5)) @@ -1162,12 +1184,12 @@ def save_results( """ if start == "": start = self.date_index[0] - else: + elif isinstance(start, str): start = dt.datetime.strptime(start, fmt) if end == "": end = self.date_index[-1] - else: + elif isinstance(end, str): end = dt.datetime.strptime(end, fmt) start_i = np.nonzero(self.date_index == start)[0][0] diff --git a/src/hapi/config.py b/src/hapi/config.py new file mode 100644 index 00000000..0b61baba --- /dev/null +++ b/src/hapi/config.py @@ -0,0 +1,417 @@ +"""Load a run configuration from YAML and assemble a `Catchment` from it. + +Every example script under `examples/hydrological-model/*/run/` starts with a block of +path/constant assignments before the `Catchment` is built and its `read_*` methods are called +in the exact order the build-then-mutate pattern (see the `hapi.catchment` module docstring) +requires. This module lifts that block into a YAML file plus a loader: `from_yaml` reads it and +assigns every input onto the `Catchment` object the same way the hand-written block did. +`load_config` is the parsing step alone, for callers that want the `RunConfig` without building +a model from it. Running the model is still the caller's job, exactly as in a hand-wired script +-- call `Run.RunHapi(model)`, `Run.runFW1(model)` or `Run.runLumped(model, ...)` yourself, +whichever `model.routing_method` / `model.spatial_resolution` calls for. + +Two spatial resolutions are supported -- lumped and distributed -- selected by +`catchment.spatial_resolution`. They disagree on the *shape* of two blocks: + +- `meteo`: a grid (`MeteoInputs`, via raster folders or NetCDF) for distributed, a single CSV + (`Catchment.read_lumped_inputs`) for lumped. +- `gauges.discharge`: a folder of one CSV per gauge id for distributed, a single CSV for + lumped. + +Lake-aware runs (`hapi.catchment.Lake`) and the flood model (`Run.RunFloodModel`) are out of +scope for this schema -- both need inputs it does not carry. + +Examples: + >>> from hapi.config import from_yaml # doctest: +SKIP + >>> from hapi.run import Run # doctest: +SKIP + >>> model = from_yaml("case-study.yaml") # doctest: +SKIP + >>> Run.RunHapi(model) # doctest: +SKIP +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Any + +import yaml + +from hapi.catchment import Catchment +from hapi.inputs import FlowNetwork, MeteoInputs +from hapi.rrm.hbv import HBV +from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 + +#: Conceptual model classes resolvable by name from `conceptual_model.model_class` in the YAML. +CONCEPTUAL_MODELS: dict[str, type] = { + "HBVBergestrom92": HBVBergestrom92, + "HBV": HBV, +} + +#: `Catchment.__init__` stores `routing_method` verbatim, with no case-folding of its own (unlike +#: `spatial_resolution` / `temporal_resolution`, which it lowercases). `distrrm.SpatialRouting` +#: -- the Muskingum routing loop `Run.RunHapi` reaches -- then compares it with +#: `Model.routing_method != "Muskingum"`, an exact, case-sensitive match against that one +#: literal. Any other spelling, including a differently-cased "muskingum", makes every cell take +#: the MAXBAS branch instead and read `Model.bankfull_depth`, which is `None` outside the flood +#: model and raises `TypeError`. MAXBAS itself never calls `SpatialRouting`, so its label is +#: cosmetic -- but Muskingum's must be this exact string. +_ROUTING_METHOD_LABELS: dict[str, str] = {"muskingum": "Muskingum", "maxbas": "MAXBAS"} + + +@dataclasses.dataclass +class CatchmentConfig: + """The `Catchment` constructor arguments. + + Attributes: + name: Catchment name. + start: Start date, parsed with `fmt`. + end: End date, parsed with `fmt`. + fmt: `strptime` format for `start` / `end`. + spatial_resolution: `"lumped"` or `"distributed"`. + temporal_resolution: `"daily"` or `"hourly"`. + routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`; + which `Run.*` entry point actually routes with it is the caller's choice. + """ + + name: str + start: str + end: str + fmt: str = "%Y-%m-%d" + spatial_resolution: str = "lumped" + temporal_resolution: str = "daily" + routing_method: str = "muskingum" + + +@dataclasses.dataclass +class MeteoConfig: + """The meteorological drivers: a distributed grid or a lumped CSV. + + Attributes: + source: `"rasters"`, `"netcdf"` or `"netcdf_files"` (distributed); ignored for lumped, + which always reads `path` as a single CSV. + precipitation: Rainfall folder (`"rasters"`), NetCDF path (`"netcdf_files"`), or the + variable name holding rainfall inside `path` (`"netcdf"`). + temperature: As `precipitation`, for temperature. + evapotranspiration: As `precipitation`, for evapotranspiration. + path: The combined NetCDF (`source="netcdf"`) or the lumped meteo CSV. + start: Optional window start; `None` uses `catchment.start`. Distributed only. + end: Optional window end; `None` uses `catchment.end`. Distributed only. + fmt: `strptime` format for `start` / `end`. + glob: Raster glob, `source="rasters"` only. + regex_string: Date regex within file names, `source="rasters"` only. + file_name_data_fmt: `strptime` format for the matched date; inferred if `None`. + per_variable: Per-folder overrides of the reader arguments, `source="rasters"` only. + gdal_env: GDAL environment overrides for the raster read, `source="rasters"` only. + """ + + source: str = "rasters" + precipitation: str | None = None + temperature: str | None = None + evapotranspiration: str | None = None + path: str | None = None + start: str | None = None + end: str | None = None + fmt: str = "%Y-%m-%d" + glob: str = "*.tif" + regex_string: str = r"\d{4}.\d{2}.\d{2}" + file_name_data_fmt: str | None = None + per_variable: dict[str, dict[str, Any]] | None = None + gdal_env: dict[str, str] | None = None + + +@dataclasses.dataclass +class FlowNetworkConfig: + """The routing network. Distributed modes only. + + Attributes: + flow_accumulation: Path to the flow-accumulation raster. + flow_direction: Path to the flow-direction raster. Required for Muskingum, unused (and + may be omitted) for MAXBAS. + """ + + flow_accumulation: str + flow_direction: str | None = None + + +@dataclasses.dataclass +class ParametersConfig: + """Where the conceptual-model parameters live. + + Attributes: + path: Folder of parameter rasters (distributed) or a CSV file (lumped). + snow: Whether the parameter set includes the snow routine (15 parameters vs 10). + maxbas: Whether the parameter set was built for MAXBAS routing. Independent of + `catchment.routing_method` -- this describes the parameter *set*, not the run. + """ + + path: str + snow: bool = False + maxbas: bool = False + + +@dataclasses.dataclass +class ConceptualModelConfig: + """The lumped conceptual model run per cell (distributed) or per catchment (lumped). + + Attributes: + model_class: Name in `CONCEPTUAL_MODELS`, e.g. `"HBVBergestrom92"`. + catchment_area: Catchment area, km2. + initial_condition: `[sp, sm, uz, lz, wc]`, five values. + q_init: Optional initial discharge; `None` derives it from the initial condition. + """ + + model_class: str + catchment_area: float + initial_condition: list[float] + q_init: float | None = None + + +@dataclasses.dataclass +class GaugesConfig: + """The observed discharge used to score the run. + + Attributes: + discharge: Folder of one CSV per gauge id (distributed) or a single CSV (lumped). + table: Gauge locations and properties. Distributed only. + column: Gauge table column holding the ids the discharge folder's file names match. + Unused for lumped. + delimiter: Discharge CSV delimiter. + fmt: `strptime` format for the discharge CSV's date column. + """ + + discharge: str + table: str | None = None + column: str = "id" + delimiter: str = "," + fmt: str = "%Y-%m-%d" + + +@dataclasses.dataclass +class OutputsConfig: + """Where to write results after the run. + + Attributes: + results_dir: Folder `save_results` writes into. + """ + + results_dir: str | None = None + + +@dataclasses.dataclass +class RunConfig: + """The full input set for one `Catchment` build. + + Attributes: + catchment: Constructor arguments. + meteo: The meteorological drivers. + parameters: Where the conceptual-model parameters live. + conceptual_model: The lumped conceptual model. + gauges: The observed discharge. + flow_network: The routing network. `None` for lumped. + outputs: Where to write results. `None` if the run is only scored in memory. + """ + + catchment: CatchmentConfig + meteo: MeteoConfig + parameters: ParametersConfig + conceptual_model: ConceptualModelConfig + gauges: GaugesConfig + flow_network: FlowNetworkConfig | None = None + outputs: OutputsConfig | None = None + + +def load_config(path: str | Path) -> RunConfig: + """Read a run configuration from a YAML file. + + Args: + path: Path to the YAML file. + + Returns: + RunConfig: The parsed configuration, not yet built into a `Catchment`. + + Raises: + ValueError: `catchment.spatial_resolution` is `"distributed"` and the file has no + `flow_network` block. + """ + raw = yaml.safe_load(Path(path).read_text()) + + catchment = CatchmentConfig(**raw["catchment"]) + is_distributed = catchment.spatial_resolution.lower() == "distributed" + + flow_network = ( + FlowNetworkConfig(**raw["flow_network"]) if "flow_network" in raw else None + ) + if is_distributed and flow_network is None: + raise ValueError( + "catchment.spatial_resolution is 'distributed' but the file has no flow_network " + "block" + ) + + return RunConfig( + catchment=catchment, + meteo=MeteoConfig(**raw["meteo"]), + parameters=ParametersConfig(**raw["parameters"]), + conceptual_model=ConceptualModelConfig(**raw["conceptual_model"]), + gauges=GaugesConfig(**raw["gauges"]), + flow_network=flow_network, + outputs=OutputsConfig(**raw["outputs"]) if "outputs" in raw else None, + ) + + +def _build_meteo(meteo: MeteoConfig, catchment: CatchmentConfig) -> MeteoInputs: + """Dispatch to the `MeteoInputs` loader `meteo.source` names. + + Args: + meteo: The meteo block. `source` must be `"rasters"`, `"netcdf"` or `"netcdf_files"`. + catchment: Supplies the default window when `meteo.start` / `meteo.end` are `None`. + + Returns: + MeteoInputs: The three cubes, windowed to the model's dates. + + Raises: + ValueError: `meteo.source` is not one of the three recognised loaders. + AssertionError: `meteo.source` names a loader whose required fields are `None` -- + `precipitation` / `temperature` / `evapotranspiration` for every source, plus + `path` for `"netcdf"`. + """ + start = meteo.start or catchment.start + end = meteo.end or catchment.end + + # precipitation/temperature/evapotranspiration are Optional on the dataclass because a + # lumped config never sets them, but every distributed source requires all three. + assert meteo.precipitation is not None, ( + "meteo.precipitation is required for a distributed run" + ) + assert meteo.temperature is not None, ( + "meteo.temperature is required for a distributed run" + ) + assert meteo.evapotranspiration is not None, ( + "meteo.evapotranspiration is required for a distributed run" + ) + + if meteo.source == "rasters": + kwargs: dict[str, Any] = dict( + glob=meteo.glob, + regex_string=meteo.regex_string, + file_name_data_fmt=meteo.file_name_data_fmt, + start=start, + end=end, + fmt=meteo.fmt, + ) + if meteo.per_variable is not None: + kwargs["per_variable"] = meteo.per_variable + if meteo.gdal_env is not None: + kwargs["gdal_env"] = meteo.gdal_env + return MeteoInputs.from_rasters( + meteo.precipitation, meteo.temperature, meteo.evapotranspiration, **kwargs + ) + + if meteo.source == "netcdf": + assert meteo.path is not None, "meteo.path is required for meteo.source: netcdf" + return MeteoInputs.from_netcdf( + meteo.path, + precipitation=meteo.precipitation, + temperature=meteo.temperature, + evapotranspiration=meteo.evapotranspiration, + start=start, + end=end, + fmt=meteo.fmt, + ) + + if meteo.source == "netcdf_files": + return MeteoInputs.from_netcdf_files( + meteo.precipitation, + meteo.temperature, + meteo.evapotranspiration, + start=start, + end=end, + fmt=meteo.fmt, + ) + + raise ValueError( + f"meteo.source must be 'rasters', 'netcdf' or 'netcdf_files' for a distributed run, " + f"got {meteo.source!r}" + ) + + +def from_yaml(path: str | Path) -> Catchment: + """Read a YAML run configuration and assemble a `Catchment` from it in one call. + + Calls `load_config(path)`, then follows the build-then-mutate pattern `hapi.catchment` + documents: constructs the model, assigns `meteo` and (distributed only) `flow_network`, + then calls the `read_*` methods in the order they depend on each other -- the same + sequence a hand-wired script's "Paths" block used to drive by hand. Running the model is + left to the caller, via whichever `Run.*` entry point (`RunHapi`, `runFW1`, `runLumped`) + fits `model.routing_method` / `model.spatial_resolution`. + + Args: + path: Path to the YAML file. + + Returns: + Catchment: The model, with every read_* call made -- gauges included. + + Raises: + ValueError: `catchment.spatial_resolution` is `"distributed"` and the file has no + `flow_network` block, or `conceptual_model.model_class` is not in + `CONCEPTUAL_MODELS`. + """ + config = load_config(path) + c = config.catchment + routing_label = _ROUTING_METHOD_LABELS.get( + c.routing_method.lower(), c.routing_method + ) + model = Catchment( + c.name, + c.start, + c.end, + fmt=c.fmt, + spatial_resolution=c.spatial_resolution, + temporal_resolution=c.temporal_resolution, + routing_method=routing_label, + ) + + is_distributed = c.spatial_resolution.lower() == "distributed" + + if is_distributed: + model.meteo = _build_meteo(config.meteo, c) + fn = config.flow_network + assert fn is not None, ( + "flow_network is required when spatial_resolution is distributed" + ) + model.flow_network = FlowNetwork.from_rasters( + fn.flow_accumulation, fn.flow_direction + ) + else: + assert config.meteo.path is not None, ( + "meteo.path is required when spatial_resolution is lumped" + ) + model.read_lumped_inputs(config.meteo.path) + + p = config.parameters + model.read_parameters(p.path, p.snow, maxbas=p.maxbas) + + cm = config.conceptual_model + model_class = CONCEPTUAL_MODELS.get(cm.model_class) + if model_class is None: + raise ValueError( + f"conceptual_model.model_class {cm.model_class!r} is not registered; known models " + f"are {sorted(CONCEPTUAL_MODELS)}" + ) + model.read_lumped_model( + model_class, cm.catchment_area, cm.initial_condition, cm.q_init + ) + + g = config.gauges + if is_distributed: + assert fn is not None, ( + "flow_network is required when spatial_resolution is distributed" + ) + assert g.table is not None, ( + "gauges.table is required when spatial_resolution is distributed" + ) + model.read_gauge_table(g.table, fn.flow_accumulation, fmt=g.fmt) + model.read_discharge_gauges( + g.discharge, delimiter=g.delimiter, column=g.column, fmt=g.fmt + ) + + return model From f953493e2622f397f99905ee42219a9d293693ec Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 01:30:30 +0200 Subject: [PATCH 03/61] style(examples): drop the unneeded quotes around results_dir YAML plain scalars admit interior spaces, so `results/saved rasters/` parses to the same string quoted or not. The quotes elsewhere in the file are load bearing and stay: an unquoted `2009-01-01` becomes a `date` rather than the `str` the config expects, and an unquoted `%Y-%m-%d` is a scanner error, `%` being a reserved directive indicator. --- .../coello/run/coello-distributed-model-run-netcdf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml index 29fb5a8b..0c4d8c0a 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml @@ -34,4 +34,4 @@ gauges: fmt: "%Y-%m-%d" outputs: - results_dir: "results/saved rasters/" + results_dir: results/saved rasters/ From 0b9604535888e2d72fd5adf29c59caa7b78b6906 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 01:40:25 +0200 Subject: [PATCH 04/61] docs(examples): drive the lumped and MAXBAS runs from YAML too Ports the remaining three Coello run scripts onto `Catchment.from_yaml`, each with its config beside it, so all four now configure a case study by editing data rather than Python. These cover the schema's variant-specific shapes, which the NetCDF example did not exercise: the lumped pair reads one CSV of catchment-average drivers instead of a grid and one discharge file instead of a gauge table plus folder, and the distributed MAXBAS run loads no flow-direction raster, since triangular routing sends every cell straight to the outlet. Routing stays in the scripts. Which function routes a lumped run is a run-time choice handed to `Run.runLumped`, not an input, so `Routing.muskingum_v` and `Routing.triangular_routing_1` are still picked there. Verified each config assembles a model field-for-field identical to the hardcoded block it replaces -- parameters, drivers, gauges, flow network and the rest -- and that all three scripts still run to completion. --- .../coello-distributed-model-run-maxbas.py | 68 +++--------- .../coello-distributed-model-run-maxbas.yaml | 34 ++++++ .../run/coello-lumped-model-run-maxbas.py | 82 +++++--------- .../run/coello-lumped-model-run-maxbas.yaml | 28 +++++ .../coello/run/coello-lumped-model-run.py | 105 +++++++----------- .../coello/run/coello-lumped-model-run.yaml | 27 +++++ 6 files changed, 175 insertions(+), 169 deletions(-) create mode 100644 examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml create mode 100644 examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml create mode 100644 examples/hydrological-model/coello/run/coello-lumped-model-run.yaml diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py index 3a8dd244..54e36969 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py @@ -1,44 +1,24 @@ -"""Distributed model with a maxbas routing scheme.""" +"""Distributed model with a maxbas routing scheme, built from YAML. -import datetime as dt +Everything that used to be a "Paths" block of hardcoded assignments now lives in +`coello-distributed-model-run-maxbas.yaml`, next to this script -- `Catchment.from_yaml` reads +it and assembles the model. Running it stays here, as in any hand-wired script. -import pandas as pd -from pyramids.dataset import Dataset +MAXBAS sends every cell straight to the outlet, so the config loads no flow-direction raster and +`extract_discharge` needs `frame_work_1=True`: a cell of `Qtot` is that cell's contribution to +the outlet rather than the discharge at it, which makes the per-gauge shortcut invalid. +""" + +from __future__ import annotations from hapi.catchment import Catchment -from hapi.inputs import FlowNetwork, MeteoInputs -from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 as HBV from hapi.run import Run -# %% Paths -Path = "examples/hydrological-model/data/distributed_model" -PrecPath = f"{Path}/prec" -Evap_Path = f"{Path}/evap" -TempPath = f"{Path}/temp" -FlowAccPath = f"{Path}/GIS/acc4000.tif" -FlowDPath = f"{Path}/GIS/fd4000.tif" -ParPath = f"{Path}/parameters_initial_maxbas" -# %% Meteorological data -AreaCoeff = 1530 -InitialCond = [0, 5, 5, 5, 0] -Snow = 0 -""" -Create the model object and read the input data -""" -start = "2009-01-01" -end = "2009-04-10" -name = "Coello" -Coello = Catchment(name, start, end, spatial_resolution="Distributed") -Coello.meteo = MeteoInputs.from_rasters( - PrecPath, TempPath, Evap_Path, file_name_data_fmt="%Y.%m.%d" +# %% Load the configuration and build the model +Coello = Catchment.from_yaml( + "examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml" ) -Coello.flow_network = FlowNetwork.from_rasters(FlowAccPath) -Coello.read_parameters(ParPath, Snow, maxbas=True) -Coello.read_lumped_model(HBV, AreaCoeff, InitialCond) -# %% Gauges -Coello.read_gauge_table(f"{Path}/stations/gauges.csv", FlowAccPath) -Coello.read_discharge_gauges(f"{Path}/stations/", column="id", fmt="%Y-%m-%d") # %% Run the model """ Outputs: @@ -59,6 +39,7 @@ 3D array of the lower zone discharge translated at each time step """ Run.runFW1(Coello) + # %% calculate performance criteria Coello.extract_discharge(calculate_metrics=True, frame_work_1=True) @@ -70,23 +51,6 @@ print("NSEhf= " + str(round(Coello.metrics.loc["NSEhf", gaugeid], 2))) print("KGE= " + str(round(Coello.metrics.loc["KGE", gaugeid], 2))) print("WB= " + str(round(Coello.metrics.loc["WB", gaugeid], 2))) -# %% plot -i = 5 -gaugei = 5 -plotstart = "2009-01-01" -plotend = "2011-12-31" - -Coello.plot_hydrograph(plotstart, plotend, gaugei) -# %% store the result into rasters -# create list of names -src = Dataset.read_file(FlowAccPath) -s = dt.datetime(2012, 6, 14, 19, 00, 00) -e = dt.datetime(2013, 12, 23, 00, 00, 00) -index = pd.date_range(s, e, freq="1H") -resultspath = "results/" -names = [resultspath + str(i)[:-6] for i in index] -names = [i.replace("-", "_") for i in names] -names = [i.replace(" ", "_") for i in names] -names = [i + ".tif" for i in names] -# Raster.RastersLike(src,q_uz_routed[:,:,:-1],names) +# %% plot the hydrograph at the outlet gauge (row position, not the gauge id) +Coello.plot_hydrograph(Coello.start, Coello.end, Coello.GaugesTable.index[-1]) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml new file mode 100644 index 00000000..3cfc45ad --- /dev/null +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml @@ -0,0 +1,34 @@ +catchment: + name: Coello + start: "2009-01-01" + end: "2009-04-10" + spatial_resolution: distributed + temporal_resolution: daily + routing_method: maxbas + +meteo: + source: rasters + precipitation: examples/hydrological-model/data/distributed_model/prec + temperature: examples/hydrological-model/data/distributed_model/temp + evapotranspiration: examples/hydrological-model/data/distributed_model/evap + file_name_data_fmt: "%Y.%m.%d" + +# MAXBAS sends every cell straight to the outlet, so no flow-direction raster is read. +flow_network: + flow_accumulation: examples/hydrological-model/data/distributed_model/GIS/acc4000.tif + +parameters: + path: examples/hydrological-model/data/distributed_model/parameters_initial_maxbas + snow: false + maxbas: true + +conceptual_model: + model_class: HBVBergestrom92 + catchment_area: 1530 + initial_condition: [0, 5, 5, 5, 0] + +gauges: + table: examples/hydrological-model/data/distributed_model/stations/gauges.csv + discharge: examples/hydrological-model/data/distributed_model/stations/ + column: id + fmt: "%Y-%m-%d" diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py index 8de7c1a7..639c467f 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py @@ -1,63 +1,40 @@ -import datetime as dt +"""Lumped model with a triangular (MAXBAS) routing scheme, built from YAML. + +Everything that used to be a "Paths" block of hardcoded assignments now lives in +`coello-lumped-model-run-maxbas.yaml`, next to this script -- `Catchment.from_yaml` reads it and +assembles the model. Running it stays here, as in any hand-wired script: the routing function is +a run-time choice rather than an input, so it is picked below and handed to `Run.runLumped`. + +The config's `parameters.maxbas: true` says the parameter file carries the triangular-routing +parameter; picking `Routing.triangular_routing_1` below is what actually routes with it. +""" -import matplotlib +from __future__ import annotations + +import datetime as dt -matplotlib.use("TkAgg") import statista.descriptors as metrics from hapi.catchment import Catchment from hapi.routing import Routing -from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 as HBVLumped from hapi.run import Run -# %% data -Parameterpath = "examples/hydrological-model/data/lumped_model/coello-lumped-parameters2022-03-13-maxbas.txt" -MeteoDataPath = "examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv" -Path = "examples/hydrological-model/data/lumped_model/" -SaveTo = "examples/hydrological-model/data/lumped_model/" -### Meteorological data -start = "2009-01-01" -end = "2011-12-31" -name = "Coello" -Coello = Catchment(name, start, end) -Coello.read_lumped_inputs(MeteoDataPath) -# %% Lumped model -# catchment area -AreaCoeff = 1530 -# [Snow pack, Soil moisture, Upper zone, Lower Zone, Water content] -InitialCond = [0, 10, 10, 10, 0] - -Coello.read_lumped_model(HBVLumped, AreaCoeff, InitialCond) -# %% ### Model Parameters -# no snow subroutine -Snow = False -Maxbas = True -Coello.read_parameters(Parameterpath, Snow, maxbas=Maxbas) -# Coello.parameters -# %% ### Observed flow -Coello.read_discharge_gauges(Path + "Qout_c.csv", fmt="%Y-%m-%d") +# %% Load the configuration and build the model +Coello = Catchment.from_yaml( + "examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml" +) + # %% Routing # RoutingFn = Routing.triangular_routing_2 RoutingFn = Routing.triangular_routing_1 Route = 1 -# %% ### Run The Model -# Coello.parameters = [1.0171762638840873, -# 358.6427125027168, -# 1.459834925116025, -# 0.2031178594731058, -# 1.0171762638840873, -# 0.7767401680547908, -# 0.24471700755374745, -# 0.03648724503470574, -# 46.41655903500876, -# 3.126313569552141, -# 1.9894177368962747] +# %% Run the model Run.runLumped(Coello, Route, RoutingFn) -# %% ### Calculate performance criteria + +# %% Calculate performance criteria scores = dict() -# gaugeid = Coello.QGauges.columns[-1] Qobs = Coello.QGauges["q"] scores["RMSE"] = metrics.rmse(Qobs, Coello.Qsim["q"]) @@ -71,17 +48,16 @@ print("NSEhf= " + str(round(scores["NSEhf"], 2))) print("KGE= " + str(round(scores["KGE"], 2))) print("WB= " + str(round(scores["WB"], 2))) -# %% ### Plot Hydrograph + +# %% Plot Hydrograph gaugei = 0 -plotstart = "2009-01-01" -plotend = "2011-12-31" -fig, ax = Coello.plot_hydrograph(plotstart, plotend, gaugei, title="Lumped Model") -# %% ### Save Results +fig, ax = Coello.plot_hydrograph(Coello.start, Coello.end, gaugei, title="Lumped Model") +# %% Save Results +SaveTo = "examples/hydrological-model/data/lumped_model/" StartDate = "2009-01-01" EndDate = "2010-04-20" -Path = f"{SaveTo}{Coello.name}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" -Coello.save_results(result=5, start=StartDate, end=EndDate, path=Path) - -# %% +path = f"{SaveTo}{Coello.name}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" +Coello.save_results(result=5, start=StartDate, end=EndDate, path=path) +print(f"results written to : {path}") diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml new file mode 100644 index 00000000..0b4bf6ea --- /dev/null +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml @@ -0,0 +1,28 @@ +catchment: + name: Coello + start: "2009-01-01" + end: "2011-12-31" + spatial_resolution: lumped + temporal_resolution: daily + +# Lumped mode reads one CSV of catchment-average drivers, not a grid: columns are +# [date, precipitation, ET, temperature], optionally followed by the long-term average. +meteo: + path: examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv + +# A single parameter file rather than a folder of rasters. `maxbas: true` says the set carries +# the triangular-routing parameter; the routing function itself is chosen at run time. +parameters: + path: examples/hydrological-model/data/lumped_model/coello-lumped-parameters2022-03-13-maxbas.txt + snow: false + maxbas: true + +conceptual_model: + model_class: HBVBergestrom92 + catchment_area: 1530 + initial_condition: [0, 10, 10, 10, 0] + +# One discharge file, and no gauge table: locating gauges on a grid is a distributed concern. +gauges: + discharge: examples/hydrological-model/data/lumped_model/Qout_c.csv + fmt: "%Y-%m-%d" diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run.py b/examples/hydrological-model/coello/run/coello-lumped-model-run.py index b21e9fb0..6d79776c 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run.py +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run.py @@ -1,86 +1,63 @@ -import datetime as dt +"""Lumped model with Muskingum routing, built from YAML. + +Everything that used to be a "Paths" block of hardcoded assignments now lives in +`coello-lumped-model-run.yaml`, next to this script -- `Catchment.from_yaml` reads it and +assembles the model. Running it stays here, as in any hand-wired script: the routing function is +a run-time choice rather than an input, so it is picked below and handed to `Run.runLumped`. + +Lumped mode reads one CSV of catchment-average drivers instead of a grid, and one discharge file +instead of a gauge table plus a folder -- see the config for both. +""" + +from __future__ import annotations -import matplotlib +import datetime as dt -matplotlib.use("TkAgg") -import statista.descriptors as PC +import statista.descriptors as metrics from hapi.catchment import Catchment from hapi.routing import Routing -from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 as HBVLumped from hapi.run import Run -# %% data -parameter_path = "examples/hydrological-model/data/lumped_model/Coello_Lumped2021-03-08_muskingum.txt" -meteo_data_path = "examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv" -path = "examples/hydrological-model/data/lumped_model/" -save_to = "examples/hydrological-model/data/lumped_model/" -### Meteorological data -start = "2009-01-01" -end = "2011-12-31" -name = "Coello" -Coello = Catchment(name, start, end) -Coello.read_lumped_inputs(meteo_data_path) -# %% Lumped model -# catchment area -AreaCoeff = 1530 -# [Snow pack, Soil moisture, Upper zone, Lower Zone, Water content] -InitialCond = [0, 10, 10, 10, 0] - -Coello.read_lumped_model(HBVLumped, AreaCoeff, InitialCond) -# %% ### Model Parameters - -Snow = False # no snow subroutine -Coello.read_parameters(parameter_path, Snow) -# Coello.parameters -# %% ### Observed flow -Coello.read_discharge_gauges(path + "Qout_c.csv", fmt="%Y-%m-%d") -# %% ### Routing +# %% Load the configuration and build the model +Coello = Catchment.from_yaml( + "examples/hydrological-model/coello/run/coello-lumped-model-run.yaml" +) +# %% Routing # RoutingFn = Routing.triangular_routing_2 RoutingFn = Routing.muskingum_v Route = 1 -# %% ### Run The Model -# Coello.parameters = [1.0171762638840873, -# 358.6427125027168, -# 1.459834925116025, -# 0.2031178594731058, -# 1.0171762638840873, -# 0.7767401680547908, -# 0.24471700755374745, -# 0.03648724503470574, -# 46.41655903500876, -# 3.126313569552141, -# 1.9894177368962747] +# %% Run the model Run.runLumped(Coello, Route, RoutingFn) -# %% ### Calculate performance criteria -# Coello.extractDischarge(OnlyOutlet=True) -metrics = dict() -# gaugeid = Coello.QGauges.columns[-1] +# %% Calculate performance criteria +scores = dict() + Qobs = Coello.QGauges["q"] -metrics["RMSE"] = PC.rmse(Qobs, Coello.Qsim["q"]) -metrics["NSE"] = PC.nse(Qobs, Coello.Qsim["q"]) -metrics["NSEhf"] = PC.nse_hf(Qobs, Coello.Qsim["q"]) -metrics["KGE"] = PC.kge(Qobs, Coello.Qsim["q"]) -metrics["WB"] = PC.wb(Qobs, Coello.Qsim["q"]) - -print("RMSE= " + str(round(metrics["RMSE"], 2))) -print("NSE= " + str(round(metrics["NSE"], 2))) -print("NSEhf= " + str(round(metrics["NSEhf"], 2))) -print("KGE= " + str(round(metrics["KGE"], 2))) -print("WB= " + str(round(metrics["WB"], 2))) -# %% ### Plot Hydrograph +scores["RMSE"] = metrics.rmse(Qobs, Coello.Qsim["q"]) +scores["NSE"] = metrics.nse(Qobs, Coello.Qsim["q"]) +scores["NSEhf"] = metrics.nse_hf(Qobs, Coello.Qsim["q"]) +scores["KGE"] = metrics.kge(Qobs, Coello.Qsim["q"]) +scores["WB"] = metrics.wb(Qobs, Coello.Qsim["q"]) + +print("RMSE= " + str(round(scores["RMSE"], 2))) +print("NSE= " + str(round(scores["NSE"], 2))) +print("NSEhf= " + str(round(scores["NSEhf"], 2))) +print("KGE= " + str(round(scores["KGE"], 2))) +print("WB= " + str(round(scores["WB"], 2))) + +# %% Plot Hydrograph gaugei = 0 -plotstart = "2009-01-01" -plotend = "2011-12-31" -fig, ax = Coello.plot_hydrograph(plotstart, plotend, gaugei, title="Lumped Model") -# %% ### Save Results +fig, ax = Coello.plot_hydrograph(Coello.start, Coello.end, gaugei, title="Lumped Model") +# %% Save Results +SaveTo = "examples/hydrological-model/data/lumped_model/" StartDate = "2009-01-01" EndDate = "2010-04-20" -path = save_to + "Results-Lumped-Model_" + str(dt.datetime.now())[0:10] + ".txt" +path = f"{SaveTo}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" Coello.save_results(result=5, start=StartDate, end=EndDate, path=path) +print(f"results written to : {path}") diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml b/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml new file mode 100644 index 00000000..a37ecb1c --- /dev/null +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml @@ -0,0 +1,27 @@ +catchment: + name: Coello + start: "2009-01-01" + end: "2011-12-31" + spatial_resolution: lumped + temporal_resolution: daily + +# Lumped mode reads one CSV of catchment-average drivers, not a grid: columns are +# [date, precipitation, ET, temperature], optionally followed by the long-term average. +meteo: + path: examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv + +# A single parameter file rather than a folder of rasters. +parameters: + path: examples/hydrological-model/data/lumped_model/Coello_Lumped2021-03-08_muskingum.txt + snow: false + maxbas: false + +conceptual_model: + model_class: HBVBergestrom92 + catchment_area: 1530 + initial_condition: [0, 10, 10, 10, 0] + +# One discharge file, and no gauge table: locating gauges on a grid is a distributed concern. +gauges: + discharge: examples/hydrological-model/data/lumped_model/Qout_c.csv + fmt: "%Y-%m-%d" From 59197e7c8abac46aee2df73ce8b435b61e292393 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 02:19:56 +0200 Subject: [PATCH 05/61] refactor(config): make config a pure schema and own the building in Catchment `hapi.config` imported `Catchment` in order to build one, so `hapi.catchment` could only reach back through an import inside the method body. The cycle was the design being wrong, not something to work around: a module that parses configuration has no business constructing models. `hapi.config` now describes data and nothing else, and imports nothing from `hapi`. Being a leaf, it is imported normally at the top of `hapi.catchment` and the local import is gone. Everything that assigns to a model -- the construction, the `MeteoInputs` / `FlowNetwork` loaders, the `read_*` order, the conceptual-model registry and the routing-label translation -- now lives in `Catchment.from_yaml`, which also drops `load_config` in favour of validating the parsed mapping where it is used. The blocks are pydantic models rather than dataclasses, which moves the rules that were assertions in the builder into the schema, where a configuration can be rejected before anything is read: - `Literal` types on `spatial_resolution`, `temporal_resolution`, `routing_method` and `meteo.source` name the accepted values in the error - `extra="forbid"` catches a misspelled key instead of dropping it - `initial_condition` must hold five states, `catchment_area` must be positive - a model validator enforces the fields each spatial resolution needs: `flow_network` and `gauges.table` for distributed, `meteo.path` for lumped, and `meteo.path` again for `source: netcdf` `from_yaml` builds `cls`, so `Run.from_yaml` and `Calibration.from_yaml` return their own type. `hapi.config.load_config` and the module-level `hapi.config.from_yaml` are gone; build a model with `Catchment.from_yaml(path)`, or validate a configuration on its own with `RunConfig.model_validate(...)`. Not a breaking change: both were added earlier on this same branch and never reached main. --- pixi.lock | 119 +++++++++++ pyproject.toml | 3 +- src/hapi/catchment.py | 197 +++++++++++++++++- src/hapi/config.py | 471 ++++++++++++++++-------------------------- 4 files changed, 488 insertions(+), 302 deletions(-) diff --git a/pixi.lock b/pixi.lock index 028b126c..8c3be14b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -49,14 +49,17 @@ environments: - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/20/c2/247c150f5ca12f8593c20e39115db551b18de5c6cb383006de21b57399e4/pyogrio-0.13.0-cp311-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/22/c0/a27bc82efc120ee8dc4bb266a0345c47cb4200b1b3fc2b57c46e2f1e23ce/pyramids_gis-0.54.0-cp314-cp314-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/36/e1/a8933a72c45a87177fbde2696e0d0755c8c9062f8c077a961c6215fa27b1/fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/4f/2f1794a3544ab9251ad2dc026fddbf4faad5785a1f23bd4a15d8649154e4/cleopatra-0.32.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/59/a8/bd530cc264e62ddbc1d1bb7225823992e6f2432c664693e9281bb6b9c359/geopandas-1.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/0d/a17e966e620545c1548125af0b29ac812dd17b197a18a7462ac12fa859ee/matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/89/2a844506d49651e9aa1af6ef95b6bd8031cb1d5a4375edec6155037e04cf/scipy-1.18.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/76/2a/d6dcec4a68ebd9056cd3bede600a5a085d21cc8df37c8b269d3570e1fc05/Oasis_Optimization-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl @@ -64,6 +67,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7f/89/648397f9936e0b330999c4e776ebf296ec3c6a65f9901687dbca4ab820da/cftime-1.6.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl @@ -72,6 +76,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/af/4d72d9e475ac83719160c662619e4bf7b95c19507cd582e7d0167a3c3dae/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl @@ -106,11 +111,13 @@ environments: - pypi: https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/4f/2f1794a3544ab9251ad2dc026fddbf4faad5785a1f23bd4a15d8649154e4/cleopatra-0.32.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/94/d73da0d28f16c45bb9b0a5691b91610b0275c5ef0eb5e43c87cf2dc1bf31/scipy-1.18.1-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/59/a8/bd530cc264e62ddbc1d1bb7225823992e6f2432c664693e9281bb6b9c359/geopandas-1.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/76/2a/d6dcec4a68ebd9056cd3bede600a5a085d21cc8df37c8b269d3570e1fc05/Oasis_Optimization-1.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl @@ -118,6 +125,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/d5/6a58eea2cb9abbb9b3f2bb8b2cfb3243d1152d69f442d256c7af71304769/scikit_learn-1.9.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/a3/36/4e551e8aa55c9188bca9abb5096805edbf7431072b76e2298e34fd3a3008/kiwisolver-1.5.0-cp314-cp314-win_amd64.whl @@ -131,9 +139,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/e1/f9/24b322756aee5798d3d1a4561c04e37128cb4ee7b2344fd601777fde35ae/pyramids_gis-0.54.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/f9/7f/d8cbabf3ba7ada509c0282a6e9c2183d01c8c1b037bba0583a5c68255e05/statista-0.8.0-py3-none-any.whl dev: channels: @@ -192,6 +202,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/5c/eb1e3ce54c4e94c7734b3831756c63f21badb3de91a98d77b9e23c0ca76a/nbval-0.11.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/32/91/30151a39f7570f448ed84529390628a651d7f27c87d73c9b887f8189695e/docutils-0.23-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl @@ -233,6 +244,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/0d/a17e966e620545c1548125af0b29ac812dd17b197a18a7462ac12fa859ee/matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -272,6 +284,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl @@ -323,6 +336,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -431,6 +445,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -469,6 +484,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl @@ -520,6 +536,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/23/f3cd1b1e5fc56517f54452c49f92049e7dd9ffc8a63de22a495581f50d04/pywinpty-3.0.5-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -536,6 +553,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f3/71/8c002223e873a870f5c41dc69b0a7c922301123e4a31d5d01ecb700aef77/jupyter_server-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/7f/d8cbabf3ba7ada509c0282a6e9c2183d01c8c1b037bba0583a5c68255e05/statista-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl @@ -595,6 +613,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl @@ -630,6 +649,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/64/0d/a17e966e620545c1548125af0b29ac812dd17b197a18a7462ac12fa859ee/matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -664,6 +684,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/fd/a40c621ff207f3ce8e484aa0fc8ba4eb6e3ecf52e15b42ba764b457a9550/editorconfig-0.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl @@ -705,6 +726,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl @@ -802,6 +824,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -835,6 +858,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/fd/a40c621ff207f3ce8e484aa0fc8ba4eb6e3ecf52e15b42ba764b457a9550/editorconfig-0.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl @@ -879,6 +903,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/23/f3cd1b1e5fc56517f54452c49f92049e7dd9ffc8a63de22a495581f50d04/pywinpty-3.0.5-cp314-cp314-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl @@ -891,6 +916,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/f3/71/8c002223e873a870f5c41dc69b0a7c922301123e4a31d5d01ecb700aef77/jupyter_server-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f6/b6/f65ac785d4ac90dcf7c831ac6256f5dd4a19780f4e1575b2c0d6eeebe319/griffelib-2.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/7f/d8cbabf3ba7ada509c0282a6e9c2183d01c8c1b037bba0583a5c68255e05/statista-0.8.0-py3-none-any.whl @@ -1451,6 +1477,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -1488,6 +1515,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl @@ -1518,6 +1546,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/77/71d78d58f15c22db16328a476426f7ac4a60d3a5a7ba3b9627ee2f7903d4/jupyter_console-6.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d1/50/d6cc4d7e508bbccf5d6027314a8312bc7ac73d0ec7f195f53838daafab40/pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl @@ -1536,6 +1565,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -1643,6 +1673,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -1680,10 +1711,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/8e/50f46a9c0ce8d2861a394c1347caae037ea0431d2f67d7feb151cbc4649a/filelock-3.32.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ad/66/0d785f0bc5e4315a96c989bb476d0fc07ea4f85132550c7b156ca2035d52/traitlets-5.16.1-py3-none-any.whl @@ -1734,6 +1767,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -1850,6 +1884,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -1891,6 +1926,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl @@ -1915,6 +1951,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/bc/8e/1239df488393d61076653bfb29f759d0f60cab8e030abdf7c17c31539b51/ipython-9.16.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bf/32/7f77ea87fa0836daab706ed5cb7f903bb25fa26a77439011aee626af11d8/ast_serialize-0.8.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c3/55/298e9b3b864a198234997e87a1471c1b17d7f3546ace6d18fb5cf1ce24b2/ipywidgets-8.1.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -1945,6 +1982,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -2048,6 +2086,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -2087,6 +2126,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl @@ -2115,6 +2155,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/e1/468ad54333e92ccb62627e62cb88e5fc14a2171daa67ed47b1b8542d5b86/tornado-6.5.8-cp39-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/ba/3757e312a98c428ac5d8b787f3608ae325174ebef6897930a42e21dd057a/pyogrio-0.13.0-cp311-abi3-win_amd64.whl @@ -2136,6 +2177,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -2254,6 +2296,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -2292,6 +2335,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl @@ -2326,6 +2370,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d4/2b/04b8a15f3a1c77bc79ddf5c73875327f34b4fa75982df2b76e45e402d364/asttokens-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/90/343bb5d0f1f9669bc252a6073f085b4abf862511bd5c9c9eaec754341f1d/wrapt-2.3.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d8/fa/ec878c28bc7f65b77e7e17af3522c9948a9711b9fa7fc4c5e3140a7e3578/decli-0.6.3-py3-none-any.whl @@ -2343,6 +2388,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -2409,6 +2455,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/23/e8/9d61dcbd1dce8ef418f06befd4ac084b4720429c26b0b1222bc218685eff/jupyter_lsp-2.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/5c/eb1e3ce54c4e94c7734b3831756c63f21badb3de91a98d77b9e23c0ca76a/nbval-0.11.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl @@ -2450,6 +2497,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl @@ -2488,6 +2536,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl @@ -2541,6 +2590,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e9/47/242f46de028074651c9bd6d8000fc340ed0d3cdd1a0eae4387826123413a/jupyterlab-4.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/7d/f50f8c478977d4aed0902e1ca558a8b457fc985937ee751304abd6f45a1f/commitizen-4.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl @@ -3375,6 +3425,7 @@ packages: - cleopatra>=0.32.0 - matplotlib>=3.11.0 - pyyaml>=6.0 + - pydantic>=2.0 - earthlens[ecmwf]>=0.12.0 ; extra == 'inputs' requires_python: '>=3.11,<4' - pypi: https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl @@ -4342,6 +4393,13 @@ packages: version: 1.3.4 sha256: 70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307 requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl + name: pydantic-core + version: 2.46.5 + sha256: 15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl name: referencing version: 0.37.0 @@ -4362,6 +4420,13 @@ packages: - ipykernel - coverage requires_python: '>=3.7,<4' +- pypi: https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.46.5 + sha256: 54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl name: overrides version: 7.7.0 @@ -7684,6 +7749,13 @@ packages: version: 3.1.1 sha256: 8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl + name: pydantic-core + version: 2.46.5 + sha256: 40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl name: pexpect version: 4.9.0 @@ -8382,6 +8454,13 @@ packages: version: 0.15.0 sha256: f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.46.5 + sha256: 0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl name: openpyxl version: 3.1.5 @@ -8649,6 +8728,20 @@ packages: requires_dist: - cached-property>=1.3.0 ; python_full_version < '3.8' requires_python: '>=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4,<4' +- pypi: https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.46.5 + sha256: 49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl + name: pydantic-core + version: 2.46.5 + sha256: 5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl name: jupyter-server-terminals version: 0.5.4 @@ -8794,6 +8887,13 @@ packages: - pytest-benchmark ; extra == 'benchmark' - geopandas ; extra == 'geopandas' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.46.5 + sha256: 6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/d4/2b/04b8a15f3a1c77bc79ddf5c73875327f34b4fa75982df2b76e45e402d364/asttokens-3.0.2-py3-none-any.whl name: asttokens version: 3.0.2 @@ -9309,6 +9409,18 @@ packages: - pyyaml-include<3.0 ; extra == 'upgrade-extension' - tomli-w<2.0 ; extra == 'upgrade-extension' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl + name: pydantic + version: 2.13.5 + sha256: 346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 + requires_dist: + - annotated-types>=0.6.0 + - pydantic-core==2.46.5 + - typing-extensions>=4.14.1 + - typing-inspection>=0.4.2 + - email-validator>=2.0.0 ; extra == 'email' + - tzdata ; python_full_version >= '3.9' and sys_platform == 'win32' and extra == 'timezone' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl name: jupyter-events version: 0.12.1 @@ -9725,6 +9837,13 @@ packages: - platformdirs>=4.2 ; extra == 'pypi' - wheel>=0.42 ; extra == 'pypi' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl + name: pydantic-core + version: 2.46.5 + sha256: fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: pillow version: 12.3.0 diff --git a/pyproject.toml b/pyproject.toml index aaeff9e5..58556142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ dependencies = [ "Oasis-Optimization >=1.0.3", "cleopatra >=0.32.0", "matplotlib >=3.11.0", - "pyyaml >=6.0" + "pyyaml >=6.0", + "pydantic >=2.0" ] [project.optional-dependencies] diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 132df5a8..6125602a 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -20,25 +20,30 @@ import os from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from pathlib import Path +from typing import TYPE_CHECKING, Any, Self import matplotlib.dates as dates import matplotlib.pyplot as plt import numpy as np import pandas as pd import statista.descriptors as metrics +import yaml from cleopatra.glyphs.gridded.array_glyph import ArrayGlyph, PointOverlay from loguru import logger from pyramids.dataset import Dataset from pyramids.dataset import DatasetCollection as Datacube from pyramids.feature import FeatureCollection +from hapi.config import CatchmentConfig, MeteoConfig, RunConfig from hapi.inputs import ( FlowNetwork, MeteoInputs, _warn_if_no_sentinel, read_rasters, ) +from hapi.rrm.hbv import HBV +from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 if TYPE_CHECKING: import matplotlib.animation @@ -55,6 +60,23 @@ (False, False): 12, } +#: Conceptual models `conceptual_model.model_class` can name in a YAML configuration. +#: `read_lumped_model` still takes any `type[BaseConceptualModel]`, so this only bounds what the +#: YAML shorthand can reach, not what the class accepts. +CONCEPTUAL_MODELS: dict[str, type[BaseConceptualModel]] = { + "HBVBergestrom92": HBVBergestrom92, + "HBV": HBV, +} + +#: `__init__` stores `routing_method` verbatim, with none of the case-folding it applies to +#: `spatial_resolution` / `temporal_resolution`. `distrrm.SpatialRouting` -- the Muskingum loop +#: `Run.RunHapi` reaches -- then tests `Model.routing_method != "Muskingum"`, an exact, +#: case-sensitive match. Any other spelling sends every cell down the MAXBAS branch, which reads +#: `bankfull_depth`: None outside the flood model, so a `TypeError`. The YAML vocabulary is +#: lower case, so it is translated here to the literal the internals expect. MAXBAS never +#: reaches that comparison, so its label is cosmetic. +_ROUTING_METHOD_LABELS = {"muskingum": "Muskingum", "maxbas": "MAXBAS"} + @contextmanager def _name_the_path(path) -> Iterator[None]: @@ -80,6 +102,63 @@ def _name_the_path(path) -> Iterator[None]: raise FileNotFoundError(f"{exc} (path: {path})") from exc +def _meteo_from_config(meteo: MeteoConfig, catchment: CatchmentConfig) -> MeteoInputs: + """Build the driver cubes with whichever `MeteoInputs` loader `meteo.source` names. + + `RunConfig` has already checked that the fields this source needs are set, so the loader is + called directly rather than re-validating here. + + Args: + meteo: The `meteo` block of a distributed configuration. + catchment: Supplies the window when `meteo.start` / `meteo.end` are unset, so the + drivers default to exactly the period the model spans. + + Returns: + MeteoInputs: The three cubes plus the calendar, windowed to the run's dates. + """ + start = meteo.start or catchment.start + end = meteo.end or catchment.end + + if meteo.source == "rasters": + extra: dict[str, Any] = {} + if meteo.per_variable is not None: + extra["per_variable"] = meteo.per_variable + if meteo.gdal_env is not None: + extra["gdal_env"] = meteo.gdal_env + return MeteoInputs.from_rasters( + meteo.precipitation, + meteo.temperature, + meteo.evapotranspiration, + glob=meteo.glob, + regex_string=meteo.regex_string, + file_name_data_fmt=meteo.file_name_data_fmt, + start=start, + end=end, + fmt=meteo.fmt, + **extra, + ) + + if meteo.source == "netcdf": + return MeteoInputs.from_netcdf( + meteo.path, + precipitation=meteo.precipitation, + temperature=meteo.temperature, + evapotranspiration=meteo.evapotranspiration, + start=start, + end=end, + fmt=meteo.fmt, + ) + + return MeteoInputs.from_netcdf_files( + meteo.precipitation, + meteo.temperature, + meteo.evapotranspiration, + start=start, + end=end, + fmt=meteo.fmt, + ) + + class Catchment: """Catchment for reading meteorological/spatial inputs and running the model. @@ -191,22 +270,120 @@ def __init__( self.metrics: pd.DataFrame | None = None @classmethod - def from_yaml(cls, path: str) -> Catchment: - """Read a YAML run configuration and assemble a `Catchment` from it. + def from_yaml(cls, path: str) -> Self: + """Read a YAML run configuration and assemble a model from it. + + The alternate constructor for the build-then-mutate pattern this class documents: it + constructs the model, assigns `meteo` and (distributed only) `flow_network`, then makes + the `read_*` calls in the order they depend on each other -- the sequence a hand-written + script's block of path assignments used to drive by hand. + + `hapi.config` only parses and validates; every assignment onto the model happens here. + Running the model stays the caller's job, through whichever `Run.*` entry point suits + `routing_method` and `spatial_resolution`. - Delegates to `hapi.config.from_yaml`, imported inside this method rather than at - module level: `hapi.config` imports `Catchment` itself (to build one), so a top-level - import here would be circular. + Builds `cls`, so `Run.from_yaml(...)` and `Calibration.from_yaml(...)` return their own + type -- both take the same constructor arguments. Args: - path: Path to the YAML file. See `hapi.config` for the schema. + path: Path to the YAML file. See :mod:`hapi.config` for the schema. Returns: - Catchment: The model, with every input read, parsed and assigned. + Self: The model, with every input read, parsed and assigned. + + Raises: + pydantic.ValidationError: The file is missing a required field, carries an unknown + one, or breaks one of the cross-field rules in :class:`hapi.config.RunConfig`. + ValueError: `conceptual_model.model_class` names a model that is not in + `CONCEPTUAL_MODELS`. + + Examples: + - Build a lumped model and inspect what the configuration gave it: + ```python + >>> from hapi.catchment import Catchment + >>> model = Catchment.from_yaml( + ... "examples/hydrological-model/coello/run/coello-lumped-model-run.yaml" + ... ) + >>> model.name + 'Coello' + >>> model.spatial_resolution + 'lumped' + >>> len(model.date_index) + 1095 + + ``` + - Build a distributed model, whose drivers and routing network come from the + `meteo` and `flow_network` blocks: + ```python + >>> from hapi.catchment import Catchment + >>> model = Catchment.from_yaml( + ... "examples/hydrological-model/coello/run/" + ... "coello-distributed-model-run-netcdf.yaml" + ... ) + >>> model.meteo.shape + (13, 14, 10) + >>> model.flow_network.rows, model.flow_network.cols + (13, 14) + >>> model.routing_method + 'Muskingum' + + ``` """ - from hapi.config import from_yaml + config = RunConfig.model_validate(yaml.safe_load(Path(path).read_text())) + catchment = config.catchment + + model = cls( + catchment.name, + catchment.start, + catchment.end, + fmt=catchment.fmt, + spatial_resolution=catchment.spatial_resolution, + temporal_resolution=catchment.temporal_resolution, + routing_method=_ROUTING_METHOD_LABELS[catchment.routing_method], + ) + + distributed = catchment.spatial_resolution == "distributed" + if distributed: + model.meteo = _meteo_from_config(config.meteo, catchment) + model.flow_network = FlowNetwork.from_rasters( + config.flow_network.flow_accumulation, + config.flow_network.flow_direction, + ) + else: + model.read_lumped_inputs(config.meteo.path) + + model.read_parameters( + config.parameters.path, + config.parameters.snow, + maxbas=config.parameters.maxbas, + ) + + conceptual_model = config.conceptual_model + if conceptual_model.model_class not in CONCEPTUAL_MODELS: + raise ValueError( + f"conceptual_model.model_class {conceptual_model.model_class!r} is not " + f"registered; known models are {sorted(CONCEPTUAL_MODELS)}" + ) + model.read_lumped_model( + CONCEPTUAL_MODELS[conceptual_model.model_class], + conceptual_model.catchment_area, + conceptual_model.initial_condition, + conceptual_model.q_init, + ) + + gauges = config.gauges + if distributed: + model.read_gauge_table( + gauges.table, config.flow_network.flow_accumulation, fmt=gauges.fmt + ) + model.read_discharge_gauges( + gauges.discharge, + delimiter=gauges.delimiter, + column=gauges.column, + fmt=gauges.fmt, + ) - return from_yaml(path) + return model def read_flow_path_length(self, path: str): """Read the flow path length raster. diff --git a/src/hapi/config.py b/src/hapi/config.py index 0b61baba..38e5b5c1 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -1,101 +1,135 @@ -"""Load a run configuration from YAML and assemble a `Catchment` from it. - -Every example script under `examples/hydrological-model/*/run/` starts with a block of -path/constant assignments before the `Catchment` is built and its `read_*` methods are called -in the exact order the build-then-mutate pattern (see the `hapi.catchment` module docstring) -requires. This module lifts that block into a YAML file plus a loader: `from_yaml` reads it and -assigns every input onto the `Catchment` object the same way the hand-written block did. -`load_config` is the parsing step alone, for callers that want the `RunConfig` without building -a model from it. Running the model is still the caller's job, exactly as in a hand-wired script --- call `Run.RunHapi(model)`, `Run.runFW1(model)` or `Run.runLumped(model, ...)` yourself, -whichever `model.routing_method` / `model.spatial_resolution` calls for. - -Two spatial resolutions are supported -- lumped and distributed -- selected by -`catchment.spatial_resolution`. They disagree on the *shape* of two blocks: - -- `meteo`: a grid (`MeteoInputs`, via raster folders or NetCDF) for distributed, a single CSV - (`Catchment.read_lumped_inputs`) for lumped. -- `gauges.discharge`: a folder of one CSV per gauge id for distributed, a single CSV for - lumped. +"""The schema of a YAML run configuration. + +This module describes data and nothing else: `RunConfig` and the blocks it nests validate a +parsed YAML mapping and hold the result. It imports nothing from `hapi`, which keeps it a leaf +of the import graph and lets `hapi.catchment` import it at module level. Reading the file and +building a model out of it -- the `Catchment` construction, the `MeteoInputs` / `FlowNetwork` +loaders, the `read_*` call order -- belongs to :meth:`hapi.catchment.Catchment.from_yaml`. + +The schema covers lumped and distributed runs, which disagree on the shape of two blocks: + +- `meteo`: a grid for distributed (raster folders or NetCDF, per `source`), and a single CSV of + catchment-average drivers for lumped. +- `gauges`: a gauge table plus a folder of per-gauge discharge files for distributed, and one + discharge file with no table for lumped. + +Which fields are required therefore depends on `catchment.spatial_resolution` and, for a +distributed run, on `meteo.source`. Those cross-field rules are enforced here by model +validators, so a `RunConfig` that validates is one the builder can consume without re-checking. Lake-aware runs (`hapi.catchment.Lake`) and the flood model (`Run.RunFloodModel`) are out of -scope for this schema -- both need inputs it does not carry. +scope -- both need inputs this schema does not carry. Examples: - >>> from hapi.config import from_yaml # doctest: +SKIP - >>> from hapi.run import Run # doctest: +SKIP - >>> model = from_yaml("case-study.yaml") # doctest: +SKIP - >>> Run.RunHapi(model) # doctest: +SKIP + - Validate a lumped configuration and read back what it holds: + ```python + >>> from hapi.config import RunConfig + >>> config = RunConfig.model_validate( + ... { + ... "catchment": { + ... "name": "Coello", + ... "start": "2009-01-01", + ... "end": "2011-12-31", + ... }, + ... "meteo": {"path": "meteo_data.csv"}, + ... "parameters": {"path": "parameters.txt"}, + ... "conceptual_model": { + ... "model_class": "HBVBergestrom92", + ... "catchment_area": 1530, + ... "initial_condition": [0, 10, 10, 10, 0], + ... }, + ... "gauges": {"discharge": "Qout_c.csv"}, + ... } + ... ) + >>> config.catchment.spatial_resolution + 'lumped' + >>> config.conceptual_model.catchment_area + 1530.0 + >>> config.gauges.fmt + '%Y-%m-%d' + + ``` + - A distributed run needs a routing network, so one without it is refused: + ```python + >>> from pydantic import ValidationError + >>> from hapi.config import RunConfig + >>> try: + ... RunConfig.model_validate( + ... { + ... "catchment": { + ... "name": "Coello", + ... "start": "2009-01-01", + ... "end": "2011-12-31", + ... "spatial_resolution": "distributed", + ... }, + ... "meteo": {"path": "meteo_data.csv"}, + ... "parameters": {"path": "parameters.txt"}, + ... "conceptual_model": { + ... "model_class": "HBVBergestrom92", + ... "catchment_area": 1530, + ... "initial_condition": [0, 10, 10, 10, 0], + ... }, + ... "gauges": {"discharge": "Qout_c.csv"}, + ... } + ... ) + ... except ValidationError as error: + ... print(error.errors()[0]["msg"]) + Value error, catchment.spatial_resolution is 'distributed', which needs a flow_network block + + ``` """ from __future__ import annotations -import dataclasses -from pathlib import Path -from typing import Any - -import yaml - -from hapi.catchment import Catchment -from hapi.inputs import FlowNetwork, MeteoInputs -from hapi.rrm.hbv import HBV -from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 +from typing import Any, Literal -#: Conceptual model classes resolvable by name from `conceptual_model.model_class` in the YAML. -CONCEPTUAL_MODELS: dict[str, type] = { - "HBVBergestrom92": HBVBergestrom92, - "HBV": HBV, -} +from pydantic import BaseModel, ConfigDict, Field, model_validator -#: `Catchment.__init__` stores `routing_method` verbatim, with no case-folding of its own (unlike -#: `spatial_resolution` / `temporal_resolution`, which it lowercases). `distrrm.SpatialRouting` -#: -- the Muskingum routing loop `Run.RunHapi` reaches -- then compares it with -#: `Model.routing_method != "Muskingum"`, an exact, case-sensitive match against that one -#: literal. Any other spelling, including a differently-cased "muskingum", makes every cell take -#: the MAXBAS branch instead and read `Model.bankfull_depth`, which is `None` outside the flood -#: model and raises `TypeError`. MAXBAS itself never calls `SpatialRouting`, so its label is -#: cosmetic -- but Muskingum's must be this exact string. -_ROUTING_METHOD_LABELS: dict[str, str] = {"muskingum": "Muskingum", "maxbas": "MAXBAS"} +#: Rejects unknown keys, so a misspelled field in the YAML fails loudly at parse time rather +#: than being dropped and surfacing later as a missing input. +_STRICT = ConfigDict(extra="forbid") -@dataclasses.dataclass -class CatchmentConfig: +class CatchmentConfig(BaseModel): """The `Catchment` constructor arguments. Attributes: name: Catchment name. - start: Start date, parsed with `fmt`. + start: Start date, parsed with `fmt`. Kept a string: the constructor does the parsing, + and an unquoted YAML date would arrive here already a `date`. end: End date, parsed with `fmt`. fmt: `strptime` format for `start` / `end`. - spatial_resolution: `"lumped"` or `"distributed"`. + spatial_resolution: `"lumped"` or `"distributed"`. Selects the shape of `meteo` and + `gauges`, and whether `flow_network` is required. temporal_resolution: `"daily"` or `"hourly"`. - routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`; - which `Run.*` entry point actually routes with it is the caller's choice. + routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`; which + `Run.*` entry point actually routes with it is the caller's choice. """ + model_config = _STRICT + name: str start: str end: str fmt: str = "%Y-%m-%d" - spatial_resolution: str = "lumped" - temporal_resolution: str = "daily" - routing_method: str = "muskingum" + spatial_resolution: Literal["lumped", "distributed"] = "lumped" + temporal_resolution: Literal["daily", "hourly"] = "daily" + routing_method: Literal["muskingum", "maxbas"] = "muskingum" -@dataclasses.dataclass -class MeteoConfig: +class MeteoConfig(BaseModel): """The meteorological drivers: a distributed grid or a lumped CSV. Attributes: - source: `"rasters"`, `"netcdf"` or `"netcdf_files"` (distributed); ignored for lumped, - which always reads `path` as a single CSV. + source: Which `MeteoInputs` loader builds the grid. Ignored for a lumped run, which + always reads `path` as a single CSV. precipitation: Rainfall folder (`"rasters"`), NetCDF path (`"netcdf_files"`), or the variable name holding rainfall inside `path` (`"netcdf"`). temperature: As `precipitation`, for temperature. evapotranspiration: As `precipitation`, for evapotranspiration. path: The combined NetCDF (`source="netcdf"`) or the lumped meteo CSV. - start: Optional window start; `None` uses `catchment.start`. Distributed only. - end: Optional window end; `None` uses `catchment.end`. Distributed only. + start: Window start; `None` falls back to `catchment.start`. Distributed only. + end: Window end; `None` falls back to `catchment.end`. Distributed only. fmt: `strptime` format for `start` / `end`. glob: Raster glob, `source="rasters"` only. regex_string: Date regex within file names, `source="rasters"` only. @@ -104,7 +138,9 @@ class MeteoConfig: gdal_env: GDAL environment overrides for the raster read, `source="rasters"` only. """ - source: str = "rasters" + model_config = _STRICT + + source: Literal["rasters", "netcdf", "netcdf_files"] = "rasters" precipitation: str | None = None temperature: str | None = None evapotranspiration: str | None = None @@ -119,66 +155,73 @@ class MeteoConfig: gdal_env: dict[str, str] | None = None -@dataclasses.dataclass -class FlowNetworkConfig: - """The routing network. Distributed modes only. +class FlowNetworkConfig(BaseModel): + """The routing network. Distributed runs only. Attributes: flow_accumulation: Path to the flow-accumulation raster. - flow_direction: Path to the flow-direction raster. Required for Muskingum, unused (and - may be omitted) for MAXBAS. + flow_direction: Path to the flow-direction raster. Muskingum needs it; MAXBAS sends + every cell straight to the outlet and never reads one, so it may be omitted. """ + model_config = _STRICT + flow_accumulation: str flow_direction: str | None = None -@dataclasses.dataclass -class ParametersConfig: +class ParametersConfig(BaseModel): """Where the conceptual-model parameters live. Attributes: - path: Folder of parameter rasters (distributed) or a CSV file (lumped). - snow: Whether the parameter set includes the snow routine (15 parameters vs 10). - maxbas: Whether the parameter set was built for MAXBAS routing. Independent of - `catchment.routing_method` -- this describes the parameter *set*, not the run. + path: Folder of parameter rasters (distributed) or a single file (lumped). + snow: Whether the parameter set includes the snow routine (15 parameters against 10). + maxbas: Whether the set carries the triangular-routing parameter. Independent of + `catchment.routing_method` -- this describes the parameter set, not the run. """ + model_config = _STRICT + path: str snow: bool = False maxbas: bool = False -@dataclasses.dataclass -class ConceptualModelConfig: - """The lumped conceptual model run per cell (distributed) or per catchment (lumped). +class ConceptualModelConfig(BaseModel): + """The lumped conceptual model, run per cell (distributed) or per catchment (lumped). Attributes: - model_class: Name in `CONCEPTUAL_MODELS`, e.g. `"HBVBergestrom92"`. + model_class: Name of the conceptual model, e.g. `"HBVBergestrom92"`. Resolved to a class + by the builder, which owns the registry of available models. catchment_area: Catchment area, km2. - initial_condition: `[sp, sm, uz, lz, wc]`, five values. - q_init: Optional initial discharge; `None` derives it from the initial condition. + initial_condition: `[sp, sm, uz, lz, wc]`, exactly five values. + q_init: Initial discharge; `None` derives it from the initial condition. """ + # `model_class` would collide with pydantic's protected `model_` namespace, so the namespace + # is cleared rather than renaming a field the YAML already uses. + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + model_class: str - catchment_area: float - initial_condition: list[float] + catchment_area: float = Field(gt=0) + initial_condition: list[float] = Field(min_length=5, max_length=5) q_init: float | None = None -@dataclasses.dataclass -class GaugesConfig: - """The observed discharge used to score the run. +class GaugesConfig(BaseModel): + """The observed discharge the run is scored against. Attributes: discharge: Folder of one CSV per gauge id (distributed) or a single CSV (lumped). - table: Gauge locations and properties. Distributed only. - column: Gauge table column holding the ids the discharge folder's file names match. - Unused for lumped. + table: Gauge locations and properties. Distributed only; a lumped run has no grid to + locate gauges on. + column: Gauge-table column holding the ids the discharge file names match. delimiter: Discharge CSV delimiter. fmt: `strptime` format for the discharge CSV's date column. """ + model_config = _STRICT + discharge: str table: str | None = None column: str = "id" @@ -186,19 +229,19 @@ class GaugesConfig: fmt: str = "%Y-%m-%d" -@dataclasses.dataclass -class OutputsConfig: +class OutputsConfig(BaseModel): """Where to write results after the run. Attributes: results_dir: Folder `save_results` writes into. """ + model_config = _STRICT + results_dir: str | None = None -@dataclasses.dataclass -class RunConfig: +class RunConfig(BaseModel): """The full input set for one `Catchment` build. Attributes: @@ -207,10 +250,12 @@ class RunConfig: parameters: Where the conceptual-model parameters live. conceptual_model: The lumped conceptual model. gauges: The observed discharge. - flow_network: The routing network. `None` for lumped. - outputs: Where to write results. `None` if the run is only scored in memory. + flow_network: The routing network. Required for distributed, absent for lumped. + outputs: Where to write results. """ + model_config = _STRICT + catchment: CatchmentConfig meteo: MeteoConfig parameters: ParametersConfig @@ -219,199 +264,43 @@ class RunConfig: flow_network: FlowNetworkConfig | None = None outputs: OutputsConfig | None = None - -def load_config(path: str | Path) -> RunConfig: - """Read a run configuration from a YAML file. - - Args: - path: Path to the YAML file. - - Returns: - RunConfig: The parsed configuration, not yet built into a `Catchment`. - - Raises: - ValueError: `catchment.spatial_resolution` is `"distributed"` and the file has no - `flow_network` block. - """ - raw = yaml.safe_load(Path(path).read_text()) - - catchment = CatchmentConfig(**raw["catchment"]) - is_distributed = catchment.spatial_resolution.lower() == "distributed" - - flow_network = ( - FlowNetworkConfig(**raw["flow_network"]) if "flow_network" in raw else None - ) - if is_distributed and flow_network is None: - raise ValueError( - "catchment.spatial_resolution is 'distributed' but the file has no flow_network " - "block" - ) - - return RunConfig( - catchment=catchment, - meteo=MeteoConfig(**raw["meteo"]), - parameters=ParametersConfig(**raw["parameters"]), - conceptual_model=ConceptualModelConfig(**raw["conceptual_model"]), - gauges=GaugesConfig(**raw["gauges"]), - flow_network=flow_network, - outputs=OutputsConfig(**raw["outputs"]) if "outputs" in raw else None, - ) - - -def _build_meteo(meteo: MeteoConfig, catchment: CatchmentConfig) -> MeteoInputs: - """Dispatch to the `MeteoInputs` loader `meteo.source` names. - - Args: - meteo: The meteo block. `source` must be `"rasters"`, `"netcdf"` or `"netcdf_files"`. - catchment: Supplies the default window when `meteo.start` / `meteo.end` are `None`. - - Returns: - MeteoInputs: The three cubes, windowed to the model's dates. - - Raises: - ValueError: `meteo.source` is not one of the three recognised loaders. - AssertionError: `meteo.source` names a loader whose required fields are `None` -- - `precipitation` / `temperature` / `evapotranspiration` for every source, plus - `path` for `"netcdf"`. - """ - start = meteo.start or catchment.start - end = meteo.end or catchment.end - - # precipitation/temperature/evapotranspiration are Optional on the dataclass because a - # lumped config never sets them, but every distributed source requires all three. - assert meteo.precipitation is not None, ( - "meteo.precipitation is required for a distributed run" - ) - assert meteo.temperature is not None, ( - "meteo.temperature is required for a distributed run" - ) - assert meteo.evapotranspiration is not None, ( - "meteo.evapotranspiration is required for a distributed run" - ) - - if meteo.source == "rasters": - kwargs: dict[str, Any] = dict( - glob=meteo.glob, - regex_string=meteo.regex_string, - file_name_data_fmt=meteo.file_name_data_fmt, - start=start, - end=end, - fmt=meteo.fmt, - ) - if meteo.per_variable is not None: - kwargs["per_variable"] = meteo.per_variable - if meteo.gdal_env is not None: - kwargs["gdal_env"] = meteo.gdal_env - return MeteoInputs.from_rasters( - meteo.precipitation, meteo.temperature, meteo.evapotranspiration, **kwargs - ) - - if meteo.source == "netcdf": - assert meteo.path is not None, "meteo.path is required for meteo.source: netcdf" - return MeteoInputs.from_netcdf( - meteo.path, - precipitation=meteo.precipitation, - temperature=meteo.temperature, - evapotranspiration=meteo.evapotranspiration, - start=start, - end=end, - fmt=meteo.fmt, - ) - - if meteo.source == "netcdf_files": - return MeteoInputs.from_netcdf_files( - meteo.precipitation, - meteo.temperature, - meteo.evapotranspiration, - start=start, - end=end, - fmt=meteo.fmt, - ) - - raise ValueError( - f"meteo.source must be 'rasters', 'netcdf' or 'netcdf_files' for a distributed run, " - f"got {meteo.source!r}" - ) - - -def from_yaml(path: str | Path) -> Catchment: - """Read a YAML run configuration and assemble a `Catchment` from it in one call. - - Calls `load_config(path)`, then follows the build-then-mutate pattern `hapi.catchment` - documents: constructs the model, assigns `meteo` and (distributed only) `flow_network`, - then calls the `read_*` methods in the order they depend on each other -- the same - sequence a hand-wired script's "Paths" block used to drive by hand. Running the model is - left to the caller, via whichever `Run.*` entry point (`RunHapi`, `runFW1`, `runLumped`) - fits `model.routing_method` / `model.spatial_resolution`. - - Args: - path: Path to the YAML file. - - Returns: - Catchment: The model, with every read_* call made -- gauges included. - - Raises: - ValueError: `catchment.spatial_resolution` is `"distributed"` and the file has no - `flow_network` block, or `conceptual_model.model_class` is not in - `CONCEPTUAL_MODELS`. - """ - config = load_config(path) - c = config.catchment - routing_label = _ROUTING_METHOD_LABELS.get( - c.routing_method.lower(), c.routing_method - ) - model = Catchment( - c.name, - c.start, - c.end, - fmt=c.fmt, - spatial_resolution=c.spatial_resolution, - temporal_resolution=c.temporal_resolution, - routing_method=routing_label, - ) - - is_distributed = c.spatial_resolution.lower() == "distributed" - - if is_distributed: - model.meteo = _build_meteo(config.meteo, c) - fn = config.flow_network - assert fn is not None, ( - "flow_network is required when spatial_resolution is distributed" - ) - model.flow_network = FlowNetwork.from_rasters( - fn.flow_accumulation, fn.flow_direction - ) - else: - assert config.meteo.path is not None, ( - "meteo.path is required when spatial_resolution is lumped" - ) - model.read_lumped_inputs(config.meteo.path) - - p = config.parameters - model.read_parameters(p.path, p.snow, maxbas=p.maxbas) - - cm = config.conceptual_model - model_class = CONCEPTUAL_MODELS.get(cm.model_class) - if model_class is None: - raise ValueError( - f"conceptual_model.model_class {cm.model_class!r} is not registered; known models " - f"are {sorted(CONCEPTUAL_MODELS)}" - ) - model.read_lumped_model( - model_class, cm.catchment_area, cm.initial_condition, cm.q_init - ) - - g = config.gauges - if is_distributed: - assert fn is not None, ( - "flow_network is required when spatial_resolution is distributed" - ) - assert g.table is not None, ( - "gauges.table is required when spatial_resolution is distributed" - ) - model.read_gauge_table(g.table, fn.flow_accumulation, fmt=g.fmt) - model.read_discharge_gauges( - g.discharge, delimiter=g.delimiter, column=g.column, fmt=g.fmt - ) - - return model + @model_validator(mode="after") + def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: + """Enforce the fields each spatial resolution requires. + + Returns: + RunConfig: This config, unchanged. + + Raises: + ValueError: A block the chosen `spatial_resolution` needs is missing, or one of the + three drivers a distributed `meteo.source` needs is unset. + """ + if self.catchment.spatial_resolution == "distributed": + if self.flow_network is None: + raise ValueError( + "catchment.spatial_resolution is 'distributed', which needs a flow_network " + "block" + ) + if self.gauges.table is None: + raise ValueError( + "catchment.spatial_resolution is 'distributed', which needs gauges.table " + "to locate the gauges on the grid" + ) + missing = [ + name + for name in ("precipitation", "temperature", "evapotranspiration") + if getattr(self.meteo, name) is None + ] + if missing: + raise ValueError( + f"a distributed run needs all three drivers; meteo is missing " + f"{', '.join(missing)}" + ) + if self.meteo.source == "netcdf" and self.meteo.path is None: + raise ValueError("meteo.source is 'netcdf', which needs meteo.path") + elif self.meteo.path is None: + raise ValueError( + "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the CSV " + "of catchment-average drivers" + ) + return self From b0f97d164d1c67268ffce49de378c8faa302d432 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 15:59:38 +0200 Subject: [PATCH 06/61] refactor(inputs): let MeteoInputs build itself from a meteo configuration The dispatch over `meteo.source` was a private helper in `hapi.catchment`, yet every line of it reached into `MeteoInputs` to choose between that class's own three loaders. It belongs on the class it envies, next to the loaders it picks from, where all four ways to construct the drivers can be read together. `hapi.config` is a leaf, so `hapi.inputs` importing it adds no cycle: config -> nothing catchment -> config, inputs, rrm.hbv* dem -> nothing run -> catchment, wrapper inputs -> config, dem `from_config` takes the fallback window as plain `start` / `end` rather than a `CatchmentConfig`, so `MeteoInputs` needs to know nothing about the catchment block. Assigning the result stays in `Catchment.from_yaml`. `hapi.inputs` is not in the mypy suppression list that covers `hapi.catchment`, so the driver fields being optional -- a lumped configuration sets none of them -- now has to be narrowed rather than ignored. The asserts document that `RunConfig` already guarantees them, and only fire for a `MeteoConfig` built by hand. `FlowNetwork` is deliberately left alone: its block maps to a single `from_rasters` call, so a `from_config` there would be an alias with no dispatch to encapsulate. --- src/hapi/catchment.py | 63 ++------------------- src/hapi/inputs.py | 126 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 59 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 6125602a..2c4f772a 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -35,7 +35,7 @@ from pyramids.dataset import DatasetCollection as Datacube from pyramids.feature import FeatureCollection -from hapi.config import CatchmentConfig, MeteoConfig, RunConfig +from hapi.config import RunConfig from hapi.inputs import ( FlowNetwork, MeteoInputs, @@ -102,63 +102,6 @@ def _name_the_path(path) -> Iterator[None]: raise FileNotFoundError(f"{exc} (path: {path})") from exc -def _meteo_from_config(meteo: MeteoConfig, catchment: CatchmentConfig) -> MeteoInputs: - """Build the driver cubes with whichever `MeteoInputs` loader `meteo.source` names. - - `RunConfig` has already checked that the fields this source needs are set, so the loader is - called directly rather than re-validating here. - - Args: - meteo: The `meteo` block of a distributed configuration. - catchment: Supplies the window when `meteo.start` / `meteo.end` are unset, so the - drivers default to exactly the period the model spans. - - Returns: - MeteoInputs: The three cubes plus the calendar, windowed to the run's dates. - """ - start = meteo.start or catchment.start - end = meteo.end or catchment.end - - if meteo.source == "rasters": - extra: dict[str, Any] = {} - if meteo.per_variable is not None: - extra["per_variable"] = meteo.per_variable - if meteo.gdal_env is not None: - extra["gdal_env"] = meteo.gdal_env - return MeteoInputs.from_rasters( - meteo.precipitation, - meteo.temperature, - meteo.evapotranspiration, - glob=meteo.glob, - regex_string=meteo.regex_string, - file_name_data_fmt=meteo.file_name_data_fmt, - start=start, - end=end, - fmt=meteo.fmt, - **extra, - ) - - if meteo.source == "netcdf": - return MeteoInputs.from_netcdf( - meteo.path, - precipitation=meteo.precipitation, - temperature=meteo.temperature, - evapotranspiration=meteo.evapotranspiration, - start=start, - end=end, - fmt=meteo.fmt, - ) - - return MeteoInputs.from_netcdf_files( - meteo.precipitation, - meteo.temperature, - meteo.evapotranspiration, - start=start, - end=end, - fmt=meteo.fmt, - ) - - class Catchment: """Catchment for reading meteorological/spatial inputs and running the model. @@ -344,7 +287,9 @@ def from_yaml(cls, path: str) -> Self: distributed = catchment.spatial_resolution == "distributed" if distributed: - model.meteo = _meteo_from_config(config.meteo, catchment) + model.meteo = MeteoInputs.from_config( + config.meteo, start=catchment.start, end=catchment.end + ) model.flow_network = FlowNetwork.from_rasters( config.flow_network.flow_accumulation, config.flow_network.flow_direction, diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index ca0a2da9..91fd5b72 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -39,6 +39,7 @@ from pyramids.feature import FeatureCollection from pyramids.netcdf import NetCDF +from hapi.config import MeteoConfig from hapi.dem import DEM @@ -1138,6 +1139,131 @@ def from_netcdf( cubes, calendar = cls._window(cubes, cls._calendar(nc), start, end, fmt) return cls(**cubes, time=calendar) + @classmethod + def from_config( + cls, + config: MeteoConfig, + start: str | None = None, + end: str | None = None, + ) -> MeteoInputs: + """Build the drivers with whichever loader the configuration's `source` names. + + The dispatch behind a `meteo` block of a YAML run configuration: `"rasters"` reads three + folders, `"netcdf_files"` one file per driver, and `"netcdf"` a single combined file + whose variables the block names. `hapi.config.RunConfig` has already checked that the + fields the chosen source needs are set, so this calls the loader directly. + + Args: + config: The `meteo` block of a distributed configuration. + start: Window start used when `config.start` is unset, so the drivers can default to + the period the model spans. `None` leaves the lower bound open. + end: Window end used when `config.end` is unset. `None` leaves it open. + + Returns: + MeteoInputs: The three cubes plus the calendar, windowed to the requested period. + + Raises: + AssertionError: A field the chosen source needs is unset. `RunConfig` rejects such a + configuration, so this only fires for a `MeteoConfig` built by hand. + + Examples: + - Load a combined NetCDF by naming the variable each driver sits in: + ```python + >>> from hapi.config import MeteoConfig + >>> from hapi.inputs import MeteoInputs + >>> meteo = MeteoInputs.from_config( + ... MeteoConfig( + ... source="netcdf", + ... path="tests/rrm/data/coello/meteo.nc", + ... precipitation="precipitation", + ... temperature="temperature", + ... evapotranspiration="evapotranspiration", + ... ) + ... ) + >>> meteo.shape + (13, 14, 10) + >>> meteo.time[0].strftime("%Y-%m-%d") + '2009-01-01' + + ``` + - Narrow the same file to part of its record with the fallback window: + ```python + >>> from hapi.config import MeteoConfig + >>> from hapi.inputs import MeteoInputs + >>> meteo = MeteoInputs.from_config( + ... MeteoConfig( + ... source="netcdf", + ... path="tests/rrm/data/coello/meteo.nc", + ... precipitation="precipitation", + ... temperature="temperature", + ... evapotranspiration="evapotranspiration", + ... ), + ... start="2009-01-03", + ... end="2009-01-07", + ... ) + >>> meteo.time_steps + 5 + >>> meteo.time[-1].strftime("%Y-%m-%d") + '2009-01-07' + + ``` + + See Also: + from_rasters: The loader `source="rasters"` dispatches to. + from_netcdf: The loader `source="netcdf"` dispatches to. + from_netcdf_files: The loader `source="netcdf_files"` dispatches to. + """ + start = config.start or start + end = config.end or end + + # Optional on the model because a lumped configuration sets none of them; every + # distributed source needs all three, which `RunConfig` enforces before this runs. + assert config.precipitation is not None, "meteo.precipitation is required" + assert config.temperature is not None, "meteo.temperature is required" + assert config.evapotranspiration is not None, ( + "meteo.evapotranspiration is required" + ) + + if config.source == "rasters": + extra: dict[str, Any] = {} + if config.per_variable is not None: + extra["per_variable"] = config.per_variable + if config.gdal_env is not None: + extra["gdal_env"] = config.gdal_env + return cls.from_rasters( + config.precipitation, + config.temperature, + config.evapotranspiration, + glob=config.glob, + regex_string=config.regex_string, + file_name_data_fmt=config.file_name_data_fmt, + start=start, + end=end, + fmt=config.fmt, + **extra, + ) + + if config.source == "netcdf": + assert config.path is not None, "meteo.path is required for source 'netcdf'" + return cls.from_netcdf( + config.path, + precipitation=config.precipitation, + temperature=config.temperature, + evapotranspiration=config.evapotranspiration, + start=start, + end=end, + fmt=config.fmt, + ) + + return cls.from_netcdf_files( + config.precipitation, + config.temperature, + config.evapotranspiration, + start=start, + end=end, + fmt=config.fmt, + ) + @staticmethod def raster_folder_to_netcdf( path: str | Path, From f7add500a750f89cf86e625bdd8da7cdfed20d22 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 16:08:35 +0200 Subject: [PATCH 07/61] fix(inputs): raise instead of asserting the fields from_config requires `assert` is stripped under `python -O`, so the guards on the driver fields were not validation at all: an incomplete `MeteoConfig` would have fallen through to the loader and failed further in, on a `None` path. Each now raises `ValueError`. The three drivers are checked together so the message names which of them the configuration leaves unset, rather than reporting only whichever assert happened to fire first, and `meteo.path` gets its own message explaining why `source: netcdf` needs it. The three are bound to locals before the check: a comprehension over `METEO_VARIABLES` reads better but mypy cannot narrow through it, and `hapi.inputs` is not in the suppression list that covers `hapi.catchment`. Verified under both `python` and `python -O` that the same `ValueError` reaches the caller. --- src/hapi/inputs.py | 51 +++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index 91fd5b72..ec4c99ea 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -1163,8 +1163,9 @@ def from_config( MeteoInputs: The three cubes plus the calendar, windowed to the requested period. Raises: - AssertionError: A field the chosen source needs is unset. `RunConfig` rejects such a - configuration, so this only fires for a `MeteoConfig` built by hand. + ValueError: A field the chosen source needs is unset -- one of the three drivers, or + `path` for `source="netcdf"`. `RunConfig` rejects such a configuration, so this + only fires for a `MeteoConfig` built by hand. Examples: - Load a combined NetCDF by naming the variable each driver sits in: @@ -1216,13 +1217,21 @@ def from_config( start = config.start or start end = config.end or end - # Optional on the model because a lumped configuration sets none of them; every - # distributed source needs all three, which `RunConfig` enforces before this runs. - assert config.precipitation is not None, "meteo.precipitation is required" - assert config.temperature is not None, "meteo.temperature is required" - assert config.evapotranspiration is not None, ( - "meteo.evapotranspiration is required" - ) + # The three are optional on the model because a lumped configuration sets none of them, + # while every distributed source needs all three. `RunConfig` enforces that, so reaching + # the raise means a `MeteoConfig` was built by hand. Bound to locals so the check both + # reports what is missing and narrows the type for the calls below. + precipitation = config.precipitation + temperature = config.temperature + evapotranspiration = config.evapotranspiration + if precipitation is None or temperature is None or evapotranspiration is None: + missing = [ + name for name in METEO_VARIABLES if getattr(config, name) is None + ] + raise ValueError( + f"MeteoInputs needs all three drivers; the configuration leaves " + f"{', '.join(missing)} unset" + ) if config.source == "rasters": extra: dict[str, Any] = {} @@ -1231,9 +1240,9 @@ def from_config( if config.gdal_env is not None: extra["gdal_env"] = config.gdal_env return cls.from_rasters( - config.precipitation, - config.temperature, - config.evapotranspiration, + precipitation, + temperature, + evapotranspiration, glob=config.glob, regex_string=config.regex_string, file_name_data_fmt=config.file_name_data_fmt, @@ -1244,21 +1253,25 @@ def from_config( ) if config.source == "netcdf": - assert config.path is not None, "meteo.path is required for source 'netcdf'" + if config.path is None: + raise ValueError( + "source 'netcdf' reads the three drivers out of one file, so the " + "configuration must set meteo.path" + ) return cls.from_netcdf( config.path, - precipitation=config.precipitation, - temperature=config.temperature, - evapotranspiration=config.evapotranspiration, + precipitation=precipitation, + temperature=temperature, + evapotranspiration=evapotranspiration, start=start, end=end, fmt=config.fmt, ) return cls.from_netcdf_files( - config.precipitation, - config.temperature, - config.evapotranspiration, + precipitation, + temperature, + evapotranspiration, start=start, end=end, fmt=config.fmt, From ece31eae68b0d2be5dc8f0c1b3aa7e49d1be6754 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 16:13:08 +0200 Subject: [PATCH 08/61] fix(catchment): raise instead of asserting the input checks `assert` is stripped under `python -O`, so seven checks in this module were not validation at all. Each now raises the exception that fits, and the three `Raises:` entries advertising `AssertionError` are corrected: - `read_lumped_model`: `q_init` and `initial_cond` type checks -> `TypeError`, naming the type that arrived - `read_parameters_bound`: unequal bound lengths -> `ValueError`, naming both - `read_discharge_gauges`: the gauge table not read yet -> `ValueError` - `save_results`: an out-of-range lumped `result` -> `ValueError`, matching what the distributed branch already raises - `Lake.__init__`: an unknown `temporal_resolution` -> `ValueError`, matching the wording `Catchment.__init__` uses - `Lake.read_lumped_model`: the `initial_condition` type check -> `TypeError` The gauge-table guard was dead as written: `__init__` sets `GaugesTable` to None, so `hasattr(self, "GaugesTable")` was always true and a caller who skipped `read_gauge_table` fell through to a `TypeError` on None a few lines later. It now tests for None and says which call is missing. `test_a_non_float_initial_discharge_is_refused` expected the `AssertionError` and now expects the `TypeError`. Verified under both `python` and `python -O` that all of them still raise. --- src/hapi/catchment.py | 51 +++++++++++++------ .../test_read_parameters_validation.py | 5 +- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 2c4f772a..690cf384 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -547,12 +547,16 @@ def read_lumped_model( self.initial_cond = initial_condition - if q_init is not None: - assert isinstance(q_init, float), "q_init should be of type float" + if q_init is not None and not isinstance(q_init, float): + raise TypeError( + f"q_init should be of type float, got {type(q_init).__name__}" + ) self.q_init = q_init - if self.initial_cond is not None: - assert isinstance(self.initial_cond, list), "init_st should be of type list" + if self.initial_cond is not None and not isinstance(self.initial_cond, list): + raise TypeError( + f"init_st should be of type list, got {type(self.initial_cond).__name__}" + ) logger.debug("Lumped model is read successfully") @@ -768,7 +772,7 @@ def read_discharge_gauges( Raises: FileNotFoundError: If the discharge file does not exist (lumped mode). - AssertionError: If the gauge table has not been read yet + ValueError: If the gauge table has not been read yet (distributed mode). """ if self.temporal_resolution.lower() == "daily": @@ -777,7 +781,14 @@ def read_discharge_gauges( ind = pd.date_range(self.start, self.end, freq="h") if self.spatial_resolution.lower() == "distributed": - assert hasattr(self, "GaugesTable"), "please read the gauges' table first" + # `__init__` sets GaugesTable to None, so the `hasattr` this replaced was always + # true and never guarded anything: a caller who skipped `read_gauge_table` got a + # `TypeError` on None a few lines down instead. + if self.GaugesTable is None: + raise ValueError( + "the gauge table has not been read yet; call read_gauge_table before " + "read_discharge_gauges in distributed mode" + ) self.QGauges = pd.DataFrame( index=ind, columns=self.GaugesTable[column].tolist() @@ -843,13 +854,15 @@ def read_parameters_bound( maxbas. Default is False. Raises: - AssertionError: If the lengths of `upper_bound` and + ValueError: If the lengths of `upper_bound` and `lower_bound` are not equal. ValueError: If `snow` is not a boolean. """ - assert len(upper_bound) == len(lower_bound), ( - "the length of UB should be the same as LB" - ) + if len(upper_bound) != len(lower_bound): + raise ValueError( + f"the length of UB should be the same as LB, got {len(upper_bound)} and " + f"{len(lower_bound)}" + ) self.UB = np.array(upper_bound) self.LB = np.array(lower_bound) @@ -1387,7 +1400,10 @@ def save_results( data[STATE_VARIABLES] = self.state_variables[start_i:end_i, :] data.to_csv(path, index=False, float_format="%.3f") else: - assert False, "the possible options are from 1 to 5" + raise ValueError( + f"in lumped mode the result parameter takes a value between 1 and 5, " + f"given: {result}" + ) logger.debug("Data is saved successfully") @@ -1432,7 +1448,10 @@ def __init__( elif temporal_resolution.lower() == "hourly": self.Index = pd.date_range(start, end, freq="h") else: - assert False, "Error" + raise ValueError( + f"available temporal resolutions are 'daily' and 'hourly', got " + f"{temporal_resolution!r}" + ) self.MeteoData: np.ndarray | None = None self.Parameters: list | None = None @@ -1504,7 +1523,7 @@ def read_lumped_model( Raises: ValueError: If `lumped_model` is not a class. - AssertionError: If `initial_condition` is not a list. + TypeError: If `initial_condition` is not a list. """ if not inspect.isclass(lumped_model): raise ValueError( @@ -1517,8 +1536,10 @@ def read_lumped_model( self.LakeArea = lake_area self.InitialCond = initial_condition - if self.InitialCond is not None: - assert isinstance(self.InitialCond, list), "init_st should be of type list" + if self.InitialCond is not None and not isinstance(self.InitialCond, list): + raise TypeError( + f"init_st should be of type list, got {type(self.InitialCond).__name__}" + ) self.Snow = snow self.OutflowCell = outflow_cell diff --git a/tests/rrm/catchment/test_read_parameters_validation.py b/tests/rrm/catchment/test_read_parameters_validation.py index 8e3fef51..79929326 100644 --- a/tests/rrm/catchment/test_read_parameters_validation.py +++ b/tests/rrm/catchment/test_read_parameters_validation.py @@ -321,11 +321,12 @@ def test_a_non_float_initial_discharge_is_refused( Test scenario: The value is divided in two inside the conceptual model, so a string or a list - fails there rather than here -- far from the call that supplied it. + fails there rather than here -- far from the call that supplied it. A `TypeError` + rather than an `AssertionError`, so the check survives `python -O`. """ model = Catchment("coello", coello_start_date, coello_end_date) - with pytest.raises(AssertionError, match="q_init should be of type float"): + with pytest.raises(TypeError, match="q_init should be of type float"): model.read_lumped_model(HBVLumped, 1530.0, coello_initial_cond, q_init=bad) From 12df630785c9e3980e2cd9ec1c2d52835dcadfb8 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 16:20:46 +0200 Subject: [PATCH 09/61] fix(run): raise instead of asserting the input-dimension checks `assert` is stripped under `python -O`, so the nineteen dimension checks guarding the distributed entry points were not validation. Each now raises `ValueError`, and the five `Raises:` entries advertising `AssertionError` are corrected. Most of them were the same checks copied across the five entry points, so the repeated ones move into two helpers rather than becoming nineteen `if` blocks: `_check_parameters_cover_grid` for the parameter rows and columns, which every distributed method asserted, and `_check_lake_meteo` for the length and column count of a lake record, asserted by both lake-aware methods. The rest -- the flow-direction grid check and the flood model's four river-geometry arrays -- are converted where they stand. Messages are unchanged, since the tests match on them. The one exception is `RunFW1withLake`, which said "three columns rain" where `runHAPIwithLake` said "three columns of rain"; sharing a helper settles it on the latter, and the test matches "three columns" either way. `test_run_validation` expected the `AssertionError` in five places and now expects the `ValueError`. Verified under both `python` and `python -O` that the checks still fire. --- src/hapi/run.py | 155 +++++++++++---------- tests/rrm/catchment/test_run_validation.py | 10 +- 2 files changed, 86 insertions(+), 79 deletions(-) diff --git a/src/hapi/run.py b/src/hapi/run.py index a220d63a..eeff4940 100644 --- a/src/hapi/run.py +++ b/src/hapi/run.py @@ -26,6 +26,47 @@ GRID_MISMATCH_ERROR = "all input data should have the same number of rows" +def _check_parameters_cover_grid(model: Catchment) -> None: + """Check the parameter array spans the catchment grid. + + The same two checks every distributed entry point makes before handing the model to the + wrapper: a parameter array smaller than the grid is indexed out of range inside the + per-cell loop, far from the call that supplied it. + + Args: + model: The model about to run, carrying `parameters` and `flow_network`. + + Raises: + ValueError: The parameter array has the wrong number of rows or columns. + """ + if np.shape(model.parameters)[0] != model.flow_network.rows: + raise ValueError(ROWS_MISMATCH_ERROR) + if np.shape(model.parameters)[1] != model.flow_network.cols: + raise ValueError(COLS_MISMATCH_ERROR) + + +def _check_lake_meteo(model: Catchment, lake: LakeType) -> None: + """Check the lake's record lines up with the distributed drivers. + + Args: + model: The model about to run, whose `meteo` sets the expected length. + lake: The lake whose `MeteoData` is checked. + + Raises: + ValueError: The lake record is a different length from the distributed drivers, or + carries fewer than the three columns the lake model reads. + """ + if np.shape(lake.MeteoData)[0] != model.meteo.time_steps: + raise ValueError( + "Lake meteorological data has to have the same length as the distributed " + "raster data" + ) + if np.shape(lake.MeteoData)[1] < 3: + raise ValueError( + "Lake Meteo data has to have at least three columns of rain, ET, and Temp" + ) + + class Run(Catchment): """Run the catchment model. @@ -67,14 +108,13 @@ def RunHapi(self): translated at each time step. Raises: - AssertionError: If input data arrays have inconsistent + ValueError: If input data arrays have inconsistent row counts, column counts, or temporal lengths. """ # input dimensions fd_rows, fd_cols = self.flow_network.flow_dir_arr.shape - assert ( - fd_rows == self.flow_network.rows and fd_cols == self.flow_network.cols - ), GRID_MISMATCH_ERROR + if fd_rows != self.flow_network.rows or fd_cols != self.flow_network.cols: + raise ValueError(GRID_MISMATCH_ERROR) # input dimensions # The three cubes already agree with each other (checked when MeteoInputs was @@ -82,13 +122,7 @@ def RunHapi(self): self.meteo.validate_against( self.flow_network.rows, self.flow_network.cols, self.date_index ) - assert np.shape(self.parameters)[0] == self.flow_network.rows, ( - ROWS_MISMATCH_ERROR - ) - assert np.shape(self.parameters)[1] == self.flow_network.cols, ( - COLS_MISMATCH_ERROR - ) - + _check_parameters_cover_grid(self) # run the model Wrapper.RRMModel(self) @@ -102,15 +136,14 @@ def RunFloodModel(self): river width, river roughness, and flood plain roughness). Raises: - AssertionError: If meteorological input arrays, parameter + ValueError: If meteorological input arrays, parameter arrays, or river geometry arrays have inconsistent dimensions. """ # input dimensions [fd_rows, fd_cols] = self.flow_network.flow_dir_arr.shape - assert ( - fd_rows == self.flow_network.rows and fd_cols == self.flow_network.cols - ), GRID_MISMATCH_ERROR + if fd_rows != self.flow_network.rows or fd_cols != self.flow_network.cols: + raise ValueError(GRID_MISMATCH_ERROR) # input dimensions # The three cubes already agree with each other (checked when MeteoInputs was @@ -118,25 +151,27 @@ def RunFloodModel(self): self.meteo.validate_against( self.flow_network.rows, self.flow_network.cols, self.date_index ) - assert np.shape(self.parameters)[0] == self.flow_network.rows, ( - ROWS_MISMATCH_ERROR - ) - assert np.shape(self.parameters)[1] == self.flow_network.cols, ( - COLS_MISMATCH_ERROR - ) - - assert ( - np.shape(self.bankfull_depth)[0] == self.flow_network.rows - and np.shape(self.river_width)[0] == self.flow_network.rows - and np.shape(self.river_roughness)[0] == self.flow_network.rows - and np.shape(self.flood_plain_roughness)[0] == self.flow_network.rows - ), GRID_MISMATCH_ERROR - assert ( - np.shape(self.bankfull_depth)[1] == self.flow_network.cols - and np.shape(self.river_width)[1] == self.flow_network.cols - and np.shape(self.river_roughness)[1] == self.flow_network.cols - and np.shape(self.flood_plain_roughness)[1] == self.flow_network.cols - ), "all input data should have the same number of columns" + _check_parameters_cover_grid(self) + if any( + np.shape(arr)[0] != self.flow_network.rows + for arr in ( + self.bankfull_depth, + self.river_width, + self.river_roughness, + self.flood_plain_roughness, + ) + ): + raise ValueError(GRID_MISMATCH_ERROR) + if any( + np.shape(arr)[1] != self.flow_network.cols + for arr in ( + self.bankfull_depth, + self.river_width, + self.river_roughness, + self.flood_plain_roughness, + ) + ): + raise ValueError("all input data should have the same number of columns") # run the model Wrapper.RRMModel(self) @@ -160,15 +195,16 @@ def runHAPIwithLake(self, lake: LakeType): rain, ET, and temperature. Raises: - AssertionError: If input data arrays have inconsistent + ValueError: If input data arrays have inconsistent dimensions or if the lake meteorological data length does not match the distributed raster data length. """ # input dimensions [fd_rows, fd_cols] = self.flow_network.flow_dir_arr.shape - assert ( - fd_rows == self.flow_network.rows and fd_cols == self.flow_network.cols - ), "all input data should have the same number of rows and columns" + if fd_rows != self.flow_network.rows or fd_cols != self.flow_network.cols: + raise ValueError( + "all input data should have the same number of rows and columns" + ) # input dimensions # The three cubes already agree with each other (checked when MeteoInputs was @@ -176,20 +212,8 @@ def runHAPIwithLake(self, lake: LakeType): self.meteo.validate_against( self.flow_network.rows, self.flow_network.cols, self.date_index ) - assert np.shape(self.parameters)[0] == self.flow_network.rows, ( - ROWS_MISMATCH_ERROR - ) - assert np.shape(self.parameters)[1] == self.flow_network.cols, ( - COLS_MISMATCH_ERROR - ) - - assert np.shape(lake.MeteoData)[0] == self.meteo.time_steps, ( - "Lake meteorological data has to have the same length as the distributed raster data" - ) - assert np.shape(lake.MeteoData)[1] >= 3, ( - "Lake Meteo data has to have at least three columns of rain, ET, and Temp" - ) - + _check_parameters_cover_grid(self) + _check_lake_meteo(self, lake) # run the model Wrapper.RRMWithlake(self, lake) @@ -216,7 +240,7 @@ def runFW1(self): shortcut is invalid for this path and raises. Raises: - AssertionError: If input data arrays have inconsistent + ValueError: If input data arrays have inconsistent row counts, column counts, or temporal lengths. """ # The three cubes already agree with each other (checked when MeteoInputs was @@ -224,13 +248,7 @@ def runFW1(self): self.meteo.validate_against( self.flow_network.rows, self.flow_network.cols, self.date_index ) - assert np.shape(self.parameters)[0] == self.flow_network.rows, ( - ROWS_MISMATCH_ERROR - ) - assert np.shape(self.parameters)[1] == self.flow_network.cols, ( - COLS_MISMATCH_ERROR - ) - + _check_parameters_cover_grid(self) # run the model Wrapper.FW1(self) @@ -267,7 +285,7 @@ def RunFW1withLake(self, lake: LakeType): is tfac and `p2[1]` is catchment area in km2. Raises: - AssertionError: If input data arrays have inconsistent + ValueError: If input data arrays have inconsistent dimensions or if the lake meteorological data length does not match the distributed raster data length. """ @@ -279,19 +297,8 @@ def RunFW1withLake(self, lake: LakeType): self.meteo.validate_against( self.flow_network.rows, self.flow_network.cols, self.date_index ) - assert np.shape(self.parameters)[0] == self.flow_network.rows, ( - ROWS_MISMATCH_ERROR - ) - assert np.shape(self.parameters)[1] == self.flow_network.cols, ( - COLS_MISMATCH_ERROR - ) - - assert np.shape(lake.MeteoData)[0] == self.meteo.time_steps, ( - "Lake meteorological data has to have the same length as the distributed raster data" - ) - assert np.shape(lake.MeteoData)[1] >= 3, ( - "Lake Meteo data has to have at least three columns rain, ET, and Temp" - ) + _check_parameters_cover_grid(self) + _check_lake_meteo(self, lake) # run the model Wrapper.FW1Withlake(self, lake) diff --git a/tests/rrm/catchment/test_run_validation.py b/tests/rrm/catchment/test_run_validation.py index 8a265932..8c54f6c9 100644 --- a/tests/rrm/catchment/test_run_validation.py +++ b/tests/rrm/catchment/test_run_validation.py @@ -137,7 +137,7 @@ def test_rejects_river_geometry_off_the_catchment_grid( _load_flat_river_geometry(coello_loaded) coello_loaded.river_width = coello_loaded.river_width[:-1, :] - with pytest.raises(AssertionError, match="number of rows"): + with pytest.raises(ValueError, match="number of rows"): Run.RunFloodModel(coello_loaded) assert "RRMModel" not in spied_wrapper, ( @@ -205,7 +205,7 @@ def test_rejects_a_lake_record_of_the_wrong_length( """ lake = _LakeStub(coello_loaded.meteo.time_steps - 1) - with pytest.raises(AssertionError, match="same length"): + with pytest.raises(ValueError, match="same length"): Run.runHAPIwithLake(coello_loaded, lake) assert "RRMWithlake" not in spied_wrapper, ( @@ -223,7 +223,7 @@ def test_rejects_a_lake_record_missing_a_column( """ lake = _LakeStub(coello_loaded.meteo.time_steps, columns=2) - with pytest.raises(AssertionError, match="three columns"): + with pytest.raises(ValueError, match="three columns"): Run.runHAPIwithLake(coello_loaded, lake) def test_rejects_a_flow_direction_grid_of_the_wrong_shape( @@ -241,7 +241,7 @@ def test_rejects_a_flow_direction_grid_of_the_wrong_shape( ) lake = _LakeStub(coello_loaded.meteo.time_steps) - with pytest.raises(AssertionError, match="rows and columns"): + with pytest.raises(ValueError, match="rows and columns"): Run.runHAPIwithLake(coello_loaded, lake) @@ -279,7 +279,7 @@ def test_rejects_parameters_off_the_catchment_grid( coello_loaded.parameters = coello_loaded.parameters[:-1, :, :] lake = _LakeStub(coello_loaded.meteo.time_steps) - with pytest.raises(AssertionError, match="as many rows as the catchment grid"): + with pytest.raises(ValueError, match="as many rows as the catchment grid"): Run.RunFW1withLake(coello_loaded, lake) assert "FW1Withlake" not in spied_wrapper, ( From 1b4c6b6e1e9e97fc59d8e571cff5a55c9f65c269 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 16:28:58 +0200 Subject: [PATCH 10/61] fix: raise instead of asserting the remaining input checks Completes the sweep: `src/hapi` now contains no `assert`, so no input check disappears under `python -O`. Twenty-three across seven modules become the exception that fits -- `TypeError` where a type is wrong, `ValueError` for a value or a length -- and the fourteen `Raises:` entries advertising `AssertionError` are corrected, including the one on the abstract `routing` in `base_model`, which declares the contract its implementations keep. Three messages were wrong where they stood and are rewritten rather than carried over: - `run_calibration` and `FW1Calibration` guarded `type(api_obj_args) is dict` with "store_history should be 0 or 1", and the solver arguments with "history_fname should be of type string". Both now say which bundle is not a dict and what arrived instead. - `lumpedCalibration` checked `basic_inputs` for 'Route' and 'RoutingFn' while reporting "should contain ['p2','init_st','UB','LB']". It now names the keys it looks for and which are missing. - The four `maxbas` checks test `>= 1` but read "has to be larger than 1". The dict checks appeared three times over, so they move into `_check_optimization_args`; that also settles `type(x) is dict` against the `isinstance` the third copy already used. Verified under both `python` and `python -O` that the checks still fire, and that the eighteen failing doctests in this package are exactly the eighteen that failed before -- the pre-existing set the doctest hook is disabled for. --- src/hapi/calibration.py | 65 ++++++++++++++++++---------- src/hapi/routing.py | 5 ++- src/hapi/rrm/base_model.py | 2 +- src/hapi/rrm/hbv.py | 34 ++++++++------- src/hapi/rrm/hbv_bergestrom92.py | 5 ++- src/hapi/rrm/hbv_lake.py | 5 ++- src/hapi/rrm/parameters.py | 72 +++++++++++++++++++------------- src/hapi/wrapper.py | 10 +++-- 8 files changed, 121 insertions(+), 77 deletions(-) diff --git a/src/hapi/calibration.py b/src/hapi/calibration.py index 850d08ba..c1e2dea6 100644 --- a/src/hapi/calibration.py +++ b/src/hapi/calibration.py @@ -26,6 +26,30 @@ ) +def _check_optimization_args(api_obj_args: Any, api_solve_args: Any) -> None: + """Check the two argument bundles the optimizer is handed are mappings. + + Both are unpacked with `**` inside Oasis, so anything else fails there rather than at the + call that supplied it. Every calibration entry point makes the same pair of checks. + + Args: + api_obj_args: Keyword arguments forwarded to the objective function. + api_solve_args: Keyword arguments forwarded to the solver. + + Raises: + TypeError: Either bundle is not a dict. + """ + if not isinstance(api_obj_args, dict): + raise TypeError( + f"the objective-function arguments should be a dict, got " + f"{type(api_obj_args).__name__}" + ) + if not isinstance(api_solve_args, dict): + raise TypeError( + f"the solver arguments should be a dict, got {type(api_solve_args).__name__}" + ) + + class Calibration(Catchment): """Calibration class for distributed hydrological model parameter optimization. @@ -93,12 +117,14 @@ def read_objective_function( objective function. If None, defaults to an empty list. Raises: - AssertionError: If objective_function is not callable. + TypeError: If objective_function is not callable. """ # check objective_function - assert callable(objective_function), ( - "The Objective function should be a function" - ) + if not callable(objective_function): + raise TypeError( + f"The Objective function should be a function, got " + f"{type(objective_function).__name__}" + ) self.objective_function = objective_function if args is None: @@ -215,15 +241,14 @@ def run_calibration( - res[1]: The optimal parameter set. Raises: - AssertionError: If input dimensions are inconsistent or if + ValueError: If input dimensions are inconsistent or if optimization arguments are not dictionaries. """ # input dimensions # [rows,cols] = self.FlowAcc.ReadAsArray().shape [fd_rows, fd_cols] = self.flow_network.flow_dir_arr.shape - assert ( - fd_rows == self.flow_network.rows and fd_cols == self.flow_network.cols - ), ROWS_MISMATCH_ERROR + if fd_rows != self.flow_network.rows or fd_cols != self.flow_network.cols: + raise ValueError(ROWS_MISMATCH_ERROR) # The three cubes already agree with each other (checked when MeteoInputs was # built); this is the other half -- that they cover the model's grid. @@ -243,8 +268,7 @@ def run_calibration( pll_type = optimization_args[1] api_solve_args = optimization_args[2] # check optimization arguement - assert type(api_obj_args) is dict, "store_history should be 0 or 1" - assert type(api_solve_args) is dict, "history_fname should be of type string " + _check_optimization_args(api_obj_args, api_solve_args) print("Calibration starts") @@ -364,7 +388,7 @@ def FW1Calibration( - res[1]: The optimal parameter set. Raises: - AssertionError: If input dimensions are inconsistent or if + ValueError: If input dimensions are inconsistent or if optimization arguments are not dictionaries. """ # input dimensions @@ -390,8 +414,7 @@ def FW1Calibration( pll_type = optimization_args[1] api_solve_args = optimization_args[2] # check optimization arguement - assert type(api_obj_args) is dict, "store_history should be 0 or 1" - assert type(api_solve_args) is dict, "history_fname should be of type string " + _check_optimization_args(api_obj_args, api_solve_args) print("Calibration starts") @@ -500,15 +523,18 @@ def lumpedCalibration( - res[1]: The optimal parameter set. Raises: - AssertionError: If `basic_inputs` is missing required keys + ValueError: If `basic_inputs` is missing required keys `"Route"` or `"RoutingFn"`, or if optimization arguments are not dictionaries. """ # basic inputs # check if all inputs are included - assert all(["Route", "RoutingFn"][i] in basic_inputs for i in range(2)), ( - "basic_inputs should contain ['p2','init_st','UB','LB'] " - ) + missing = [key for key in ("Route", "RoutingFn") if key not in basic_inputs] + if missing: + raise ValueError( + f"basic_inputs should contain 'Route' and 'RoutingFn'; " + f"{', '.join(missing)} is missing" + ) route = basic_inputs["Route"] routing_fn = basic_inputs["RoutingFn"] @@ -524,10 +550,7 @@ def lumpedCalibration( pll_type = optimization_args[1] api_solve_args = optimization_args[2] # check optimization arguement - assert isinstance(api_obj_args, dict), "store_history should be 0 or 1" - assert isinstance(api_solve_args, dict), ( - "history_fname should be of type string " - ) + _check_optimization_args(api_obj_args, api_solve_args) print("Calibration starts") diff --git a/src/hapi/routing.py b/src/hapi/routing.py index a052cc0f..9e3c7796 100644 --- a/src/hapi/routing.py +++ b/src/hapi/routing.py @@ -207,7 +207,7 @@ def triangular_routing_2(q, maxbas=1): length as ``q``. Raises: - AssertionError: If ``maxbas`` is less than 1. + ValueError: If ``maxbas`` is less than 1. Examples: >>> import numpy as np @@ -216,7 +216,8 @@ def triangular_routing_2(q, maxbas=1): >>> q_routed = Routing.triangular_routing_2(q, maxbas=3) """ # input data validation - assert maxbas >= 1, "Maxbas value has to be larger than 1" + if maxbas < 1: + raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas maxbas = int(round(maxbas, 0)) diff --git a/src/hapi/rrm/base_model.py b/src/hapi/rrm/base_model.py index 281457a2..02190a3e 100644 --- a/src/hapi/rrm/base_model.py +++ b/src/hapi/rrm/base_model.py @@ -272,7 +272,7 @@ def routing(self, q: np.ndarray, maxbas: int = 1) -> np.ndarray: length as ``q``. Raises: - AssertionError: If ``maxbas`` is less than 1. + ValueError: If ``maxbas`` is less than 1. Examples: >>> import numpy as np diff --git a/src/hapi/rrm/hbv.py b/src/hapi/rrm/hbv.py index 7c4b9959..1312dc5f 100644 --- a/src/hapi/rrm/hbv.py +++ b/src/hapi/rrm/hbv.py @@ -485,7 +485,7 @@ def routing(self, q, maxbas=1): length as ``q``. Raises: - AssertionError: If ``maxbas`` is less than 1. + ValueError: If ``maxbas`` is less than 1. Examples: >>> import numpy as np @@ -496,7 +496,8 @@ def routing(self, q, maxbas=1): >>> print(q_routed.round(4)) [0. 0. 2.5 4. 2.5 0.5] """ - assert maxbas >= 1, "Maxbas value has to be larger than 1" + if maxbas < 1: + raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas # maxbas = int(maxbas) maxbas = int(round(maxbas, 0)) @@ -556,7 +557,7 @@ def step_run( updated state variables ``[sp, sm, uz, lz, wc]``. Raises: - AssertionError: If ``snow=1`` and the parameter vector + ValueError: If ``snow=1`` and the parameter vector does not have 18 elements. Examples: @@ -580,10 +581,11 @@ def step_run( ## Parse of parameters from input vector to model # picipitation function if snow == 1: - assert len(p) == 18, ( - "current version of HBV (with snow) takes 18 parameter you have entered " - + str(len(p)) - ) + if len(p) != 18: + raise ValueError( + "current version of HBV (with snow) takes 18 parameter you have " + f"entered {len(p)}" + ) ltt = p[0] utt = p[1] rfcf = p[2] @@ -732,7 +734,7 @@ def simulate( ``(n+1, 5)`` (float32). Raises: - AssertionError: If ``init_st`` does not have 5 elements + ValueError: If ``init_st`` does not have 5 elements or if ``snow`` is not 0 or 1. Examples: @@ -758,15 +760,17 @@ def simulate( q_uz length=6, first=0.0727 """ # data type - assert len(init_st) == 5, ( - "state variables are 5 and the given initial values are " - + str(len(init_st)) - ) + if len(init_st) != 5: + raise ValueError( + f"state variables are 5 and the given initial values are {len(init_st)}" + ) # assert type(p2) == list, " p2 should be of type list" # assert len(p2) == 2, "p2 should contains tfac and catchment area" - assert snow == 0 or snow == 1, ( - " snow input defines whether to consider snow subroutine or not it has to be 0 or 1" - ) + if snow not in (0, 1): + raise ValueError( + "snow input defines whether to consider snow subroutine or not it has " + f"to be 0 or 1, got {snow}" + ) if init_st is None: # 0 1 2 3 4 5 st = [DEF_ST] # [sp,sm,uz,lz,wc,LA] diff --git a/src/hapi/rrm/hbv_bergestrom92.py b/src/hapi/rrm/hbv_bergestrom92.py index 6dcce656..3e6dffcf 100644 --- a/src/hapi/rrm/hbv_bergestrom92.py +++ b/src/hapi/rrm/hbv_bergestrom92.py @@ -422,7 +422,7 @@ def routing(self, q, maxbas=1): shape as ``q``. Raises: - AssertionError: If ``maxbas`` is less than 1. + ValueError: If ``maxbas`` is less than 1. Examples: >>> from hapi.rrm.hbv_bergestrom92 import HBVBergestrom92 @@ -433,7 +433,8 @@ def routing(self, q, maxbas=1): >>> len(q_r) == len(q) True """ - assert maxbas >= 1, "Maxbas value has to be larger than 1" + if maxbas < 1: + raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas maxbas = int(round(maxbas, 0)) diff --git a/src/hapi/rrm/hbv_lake.py b/src/hapi/rrm/hbv_lake.py index 6fc0e062..cefae98d 100644 --- a/src/hapi/rrm/hbv_lake.py +++ b/src/hapi/rrm/hbv_lake.py @@ -483,7 +483,7 @@ def _routing(self, q, maxbas=1): length as ``q``. Raises: - AssertionError: If ``maxbas`` < 1. + ValueError: If ``maxbas`` < 1. Examples: >>> import numpy as np @@ -493,7 +493,8 @@ def _routing(self, q, maxbas=1): >>> len(q_r) == len(q) True """ - assert maxbas >= 1, "Maxbas value has to be larger than 1" + if maxbas < 1: + raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas maxbas = int(round(maxbas, 0)) diff --git a/src/hapi/rrm/parameters.py b/src/hapi/rrm/parameters.py index 19b01925..ddce27bf 100644 --- a/src/hapi/rrm/parameters.py +++ b/src/hapi/rrm/parameters.py @@ -87,7 +87,7 @@ def __init__( ValueError: If `function` is not one of the ints 1, 2, 3 or 4. A `bool`, a `float` such as `2.0`, and an unhashable value are all rejected rather than coerced or allowed to raise `TypeError`. - AssertionError: If `no_parameters` is not an integer, if + TypeError: If `no_parameters` is not an integer, if `no_lumped_par` is not an integer, or if the length of `lumped_par_pos` does not match `no_lumped_par`. ValueError: If `lumped_par_pos` is not a list when @@ -149,17 +149,24 @@ def __init__( raise TypeError( "raster should be a pyramids Dataset, read it using pyramids.dataset.Dataset.read_file" ) - assert isinstance(no_parameters, int), " no_parameters should be integer number" - assert isinstance(no_lumped_par, int), ( - "no of lumped parameters should be integer" - ) + if not isinstance(no_parameters, int): + raise TypeError( + f"no_parameters should be integer number, got " + f"{type(no_parameters).__name__}" + ) + if not isinstance(no_lumped_par, int): + raise TypeError( + f"no of lumped parameters should be integer, got " + f"{type(no_lumped_par).__name__}" + ) if no_lumped_par >= 1: if isinstance(lumped_par_pos, list): - assert no_lumped_par == len(lumped_par_pos), ( - f"you have to entered {no_lumped_par} no of lumped parameters but only {len(lumped_par_pos)} " - f"position " - ) + if no_lumped_par != len(lumped_par_pos): + raise ValueError( + f"you have to entered {no_lumped_par} no of lumped parameters " + f"but only {len(lumped_par_pos)} position " + ) else: # if not int or list raise ValueError( "you have one or more lumped parameters, so the position has to be entered as a list" @@ -289,7 +296,7 @@ def par3d(self, par_g: list | np.ndarray): # , kub=1,klb=0.5, Maskingum=True lumped parameter values should be appended at the end. Raises: - AssertionError: If the length of `par_g` does not match the + ValueError: If the length of `par_g` does not match the expected number of parameters based on the number of elements and lumped parameters. """ @@ -303,20 +310,25 @@ def par3d(self, par_g: list | np.ndarray): # , kub=1,klb=0.5, Maskingum=True if self.no_lumped_par > 0: par_no = (self.no_elem * self.no_parameters) + self.no_lumped_par - assert len(par_g) == par_no, ( - f"As there is {self.no_lumped_par} lumped parameters, length of input parameters should be " - f"{self.no_elem}" - + f"*({self.no_parameters + self.no_lumped_par} - {self.no_lumped_par}) + {self.no_lumped_par} = " - + f"{self.no_elem * (self.no_parameters - self.no_lumped_par) + self.no_lumped_par} not {len(par_g)}" - + " probably you have to add the value of the lumped parameter at the end of the list" - ) + if len(par_g) != par_no: + raise ValueError( + f"As there is {self.no_lumped_par} lumped parameters, length of " + f"input parameters should be {self.no_elem}" + f"*({self.no_parameters + self.no_lumped_par} - " + f"{self.no_lumped_par}) + {self.no_lumped_par} = " + f"{self.no_elem * (self.no_parameters - self.no_lumped_par) + self.no_lumped_par}" + f" not {len(par_g)} probably you have to add the value of the " + f"lumped parameter at the end of the list" + ) else: # if there are no lumped parameters par_no = self.no_elem * self.no_parameters - assert len(par_g) == par_no, ( - f"As there is no lumped parameters length of input parameters should be {self.no_elem} * " - + f"{self.no_parameters} = {self.no_elem * self.no_parameters}" - ) + if len(par_g) != par_no: + raise ValueError( + f"As there is no lumped parameters length of input parameters " + f"should be {self.no_elem} * {self.no_parameters} = " + f"{self.no_elem * self.no_parameters}" + ) # parameters in array # create a 2d array [no_parameters, no_cells] @@ -523,7 +535,7 @@ def hydrologic_response_units(self, par_g: list | np.ndarray): # ,kub=1,klb=0.5 ValueError: If `par_g` is not a numpy ndarray or a list, or if the length of `par_g` does not match the expected number of parameters. - AssertionError: If there are lumped parameters and the length + ValueError: If there are lumped parameters and the length of `par_g` does not match the expected total. """ # input data validation @@ -536,15 +548,15 @@ def hydrologic_response_units(self, par_g: list | np.ndarray): # ,kub=1,klb=0.5 # input values if self.no_lumped_par > 0: par_no = (self.no_elem * self.no_parameters) + self.no_lumped_par - assert len(par_g) == par_no, ( - f"As there is {self.no_lumped_par} lumped parameters, length of input parameters should be " - f"{self.no_elem}*({self.no_parameters}-{self.no_lumped_par})+{self.no_lumped_par}=" - + str( - self.no_elem * (self.no_parameters - self.no_lumped_par) - + self.no_lumped_par + if len(par_g) != par_no: + raise ValueError( + f"As there is {self.no_lumped_par} lumped parameters, length of " + f"input parameters should be {self.no_elem}*" + f"({self.no_parameters}-{self.no_lumped_par})+{self.no_lumped_par}=" + f"{self.no_elem * (self.no_parameters - self.no_lumped_par) + self.no_lumped_par}" + f" not {len(par_g)} probably you have to add the value of the " + f"lumped parameter at the end of the list" ) - + f" not {len(par_g)} probably you have to add the value of the lumped parameter at the end of the list" - ) else: # if there is no lumped parameters if not len(par_g) == self.no_elem * self.no_parameters: diff --git a/src/hapi/wrapper.py b/src/hapi/wrapper.py index adf0c375..ad1aa8b0 100644 --- a/src/hapi/wrapper.py +++ b/src/hapi/wrapper.py @@ -390,14 +390,16 @@ def Lumped(Model: Catchment, Routing: int = 0, RoutingFn: Callable | None = None discharge hydrograph. Must be callable. Raises: - AssertionError: If `RoutingFn` is not callable when + TypeError: If `RoutingFn` is not callable when routing is enabled. """ ### input data validation if Routing != 0: - assert callable(RoutingFn), ( - "routing function should be of type callable (function that takes arguments)" - ) + if not callable(RoutingFn): + raise TypeError( + "routing function should be of type callable (function that takes " + f"arguments), got {type(RoutingFn).__name__}" + ) # data p = Model.data[:, 0] From 08c96c3c89a213edd411fa60a5f488144c7f0286 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 17:50:53 +0200 Subject: [PATCH 11/61] feat(config): make the parameters and gauges blocks optional `from_yaml` assumed every model reads a fitted parameter set and is scored against gauges, but neither holds generally: - a calibration derives its parameters from the bounds handed to `read_parameters_bound`, so it never calls `read_parameters` and has no parameter path to name - a run that is not scored against observations has no gauge data at all Both blocks are now optional, and the builder skips the corresponding `read_*` call when one is absent. The distributed check on `gauges.table` applies only when the block is present, since it exists to catch a gauge table missing from a configuration that does carry gauges. --- src/hapi/catchment.py | 33 +++++++++++++++++++-------------- src/hapi/config.py | 14 +++++++++----- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 690cf384..54488169 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -297,11 +297,14 @@ def from_yaml(cls, path: str) -> Self: else: model.read_lumped_inputs(config.meteo.path) - model.read_parameters( - config.parameters.path, - config.parameters.snow, - maxbas=config.parameters.maxbas, - ) + # A calibration derives its parameters from the bounds `read_parameters_bound` is + # given rather than reading a fitted set, so the block is optional. + if config.parameters is not None: + model.read_parameters( + config.parameters.path, + config.parameters.snow, + maxbas=config.parameters.maxbas, + ) conceptual_model = config.conceptual_model if conceptual_model.model_class not in CONCEPTUAL_MODELS: @@ -316,17 +319,19 @@ def from_yaml(cls, path: str) -> Self: conceptual_model.q_init, ) + # Equally optional: a run that is not scored against observations has no gauges. gauges = config.gauges - if distributed: - model.read_gauge_table( - gauges.table, config.flow_network.flow_accumulation, fmt=gauges.fmt + if gauges is not None: + if distributed: + model.read_gauge_table( + gauges.table, config.flow_network.flow_accumulation, fmt=gauges.fmt + ) + model.read_discharge_gauges( + gauges.discharge, + delimiter=gauges.delimiter, + column=gauges.column, + fmt=gauges.fmt, ) - model.read_discharge_gauges( - gauges.discharge, - delimiter=gauges.delimiter, - column=gauges.column, - fmt=gauges.fmt, - ) return model diff --git a/src/hapi/config.py b/src/hapi/config.py index 38e5b5c1..ca78ef0d 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -247,9 +247,11 @@ class RunConfig(BaseModel): Attributes: catchment: Constructor arguments. meteo: The meteorological drivers. - parameters: Where the conceptual-model parameters live. conceptual_model: The lumped conceptual model. - gauges: The observed discharge. + parameters: Where the conceptual-model parameters live. Omit for a calibration, which + derives them from the bounds handed to `read_parameters_bound` rather than reading + a fitted set. + gauges: The observed discharge. Omit for a run that is not scored against gauges. flow_network: The routing network. Required for distributed, absent for lumped. outputs: Where to write results. """ @@ -258,9 +260,9 @@ class RunConfig(BaseModel): catchment: CatchmentConfig meteo: MeteoConfig - parameters: ParametersConfig conceptual_model: ConceptualModelConfig - gauges: GaugesConfig + parameters: ParametersConfig | None = None + gauges: GaugesConfig | None = None flow_network: FlowNetworkConfig | None = None outputs: OutputsConfig | None = None @@ -281,7 +283,9 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: "catchment.spatial_resolution is 'distributed', which needs a flow_network " "block" ) - if self.gauges.table is None: + # Only when gauges are configured at all: a distributed run that is not scored + # against observations omits the block entirely. + if self.gauges is not None and self.gauges.table is None: raise ValueError( "catchment.spatial_resolution is 'distributed', which needs gauges.table " "to locate the gauges on the grid" From 598d38676f40408dbde23d620c8e4f0f6cbbafb4 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 18:22:41 +0200 Subject: [PATCH 12/61] test(config): cover the run-configuration schema and the model it builds `hapi.config` and `Catchment.from_yaml` arrived on this branch with no unit tests: the schema sat at 72% and the builder was not exercised at all, its happy path covered only by a doctest. Forty tests, one class per block plus one for the builder. The emphasis is on the cross-field rules, since which blocks a configuration needs depends on `spatial_resolution` and `meteo.source` -- each is pinned separately, with the message it produces, so a rule that stops firing says which one. The builder is covered on both shapes: the distributed path down to every populated attribute, and the lumped path, which reads one averaged-driver CSV and skips the gauge table it has no grid to locate gauges on. `hapi.config` goes 72% -> 100% line and branch. The suite also caught a false claim in the `from_yaml` docstring. It said `Run.from_yaml` returns a `Run`, but `Run` overrides `__init__` to take only `self` -- its entry points are called unbound on a catchment -- so the call raises `TypeError` at the constructor. The docstring now says so, and a test pins the behaviour rather than the wish. --- src/hapi/catchment.py | 6 +- tests/test_config.py | 759 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 763 insertions(+), 2 deletions(-) create mode 100644 tests/test_config.py diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 54488169..11853f55 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -225,8 +225,10 @@ def from_yaml(cls, path: str) -> Self: Running the model stays the caller's job, through whichever `Run.*` entry point suits `routing_method` and `spatial_resolution`. - Builds `cls`, so `Run.from_yaml(...)` and `Calibration.from_yaml(...)` return their own - type -- both take the same constructor arguments. + Builds `cls`, so `Calibration.from_yaml(...)` returns a `Calibration` -- it takes the + same constructor arguments. `Run` does not: it overrides `__init__` to take none, and + its entry points are called unbound on a catchment (`Run.RunHapi(model)`), so + `Run.from_yaml` raises `TypeError` rather than silently building the wrong thing. Args: path: Path to the YAML file. See :mod:`hapi.config` for the schema. diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..1eff65df --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,759 @@ +"""Tests for the YAML run-configuration schema and the model it builds. + +`hapi.config` is pure data: pydantic models that validate a parsed YAML mapping. The rules that +matter are the cross-field ones, because which blocks a configuration needs depends on +`catchment.spatial_resolution` and on `meteo.source` -- a distributed run needs a routing +network and all three drivers, a lumped one needs the single averaged-driver CSV. Those rules +exist so `Catchment.from_yaml` can consume a validated config without re-checking anything, so +they are pinned here per rule rather than in aggregate. + +The second half covers `Catchment.from_yaml` itself: that it makes the `read_*` calls in the +order the build-then-mutate pattern requires, that it skips the two optional blocks when they +are absent, and that it builds `cls` so the subclasses return their own type. +""" + +from __future__ import annotations + +import copy + +import pytest +import yaml +from pydantic import ValidationError + +from hapi.calibration import Calibration +from hapi.catchment import Catchment +from hapi.config import ( + CatchmentConfig, + ConceptualModelConfig, + FlowNetworkConfig, + GaugesConfig, + MeteoConfig, + OutputsConfig, + ParametersConfig, + RunConfig, +) +from hapi.run import Run + +COMBINED_NC = "tests/rrm/data/coello/meteo.nc" + + +@pytest.fixture(scope="function") +def distributed_mapping( + coello_start_date: str, + coello_end_date: str, + coello_acc_path: str, + coello_fd_path: str, + coello_dist_parameters_muskingum: str, + coello_cat_area: int, + coello_initial_cond: list, + coello_gauges_table: str, + coello_gauges_path: str, +) -> dict: + """A complete distributed configuration, as a plain mapping. + + Returns: + dict: A mapping that validates, for tests to mutate one field at a time. + """ + return { + "catchment": { + "name": "Coello", + "start": coello_start_date, + "end": coello_end_date, + "spatial_resolution": "distributed", + }, + "meteo": { + "source": "netcdf", + "path": COMBINED_NC, + "precipitation": "precipitation", + "temperature": "temperature", + "evapotranspiration": "evapotranspiration", + }, + "conceptual_model": { + "model_class": "HBVBergestrom92", + "catchment_area": coello_cat_area, + "initial_condition": coello_initial_cond, + }, + "parameters": {"path": coello_dist_parameters_muskingum, "snow": False}, + "gauges": {"table": coello_gauges_table, "discharge": coello_gauges_path}, + "flow_network": { + "flow_accumulation": coello_acc_path, + "flow_direction": coello_fd_path, + }, + } + + +@pytest.fixture(scope="function") +def lumped_mapping( + coello_start_date: str, + coello_end_date: str, + lumped_meteo_data_path: str, + lumped_parameters_path: str, + lumped_gauges_path: str, +) -> dict: + """A complete lumped configuration, as a plain mapping. + + Points at the bundled lumped fixtures, so the mapping both validates and builds. + + Returns: + dict: A mapping that validates and can be handed to `from_yaml`. + """ + return { + "catchment": { + "name": "Coello", + "start": coello_start_date, + "end": coello_end_date, + "spatial_resolution": "lumped", + }, + "meteo": {"path": lumped_meteo_data_path}, + "conceptual_model": { + "model_class": "HBVBergestrom92", + "catchment_area": 1530, + "initial_condition": [0, 10, 10, 10, 0], + }, + "parameters": {"path": lumped_parameters_path}, + "gauges": {"discharge": lumped_gauges_path}, + } + + +def write_yaml(mapping: dict, tmp_path) -> str: + """Dump a mapping to a YAML file. + + Args: + mapping: The configuration to write. + tmp_path: pytest temporary directory. + + Returns: + str: Path to the written file. + """ + path = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump(mapping), encoding="utf-8") + return str(path) + + +class TestCatchmentConfig: + """Tests for the `catchment` block.""" + + def test_defaults_fill_the_optional_fields(self): + """Test that only name and the two dates are required. + + Test scenario: + The remaining fields describe the common case -- a daily lumped run parsed with + ISO dates -- so a caller should not have to restate them. + """ + config = CatchmentConfig(name="Coello", start="2009-01-01", end="2009-01-10") + + assert config.fmt == "%Y-%m-%d", f"unexpected default fmt: {config.fmt}" + assert config.spatial_resolution == "lumped", ( + f"expected 'lumped' by default, got {config.spatial_resolution}" + ) + assert config.temporal_resolution == "daily", ( + f"expected 'daily' by default, got {config.temporal_resolution}" + ) + assert config.routing_method == "muskingum", ( + f"expected 'muskingum' by default, got {config.routing_method}" + ) + + @pytest.mark.parametrize( + "field, value, allowed", + [ + ("spatial_resolution", "semi", "'lumped' or 'distributed'"), + ("temporal_resolution", "weekly", "'daily' or 'hourly'"), + ("routing_method", "kinematic", "'muskingum' or 'maxbas'"), + ], + ids=["spatial", "temporal", "routing"], + ) + def test_an_unknown_value_names_the_accepted_ones(self, field, value, allowed): + """Test that each enumerated field rejects an unknown value and lists the valid set. + + Args: + field: The field under test. + value: An unrecognised value for it. + allowed: The wording the error is expected to carry. + + Test scenario: + These three select whole code paths downstream, so a typo has to fail at parse + time naming what was expected, rather than selecting a branch by accident. + """ + kwargs = { + "name": "Coello", + "start": "2009-01-01", + "end": "2009-01-10", + field: value, + } + + with pytest.raises(ValidationError, match="Input should be") as exc: + CatchmentConfig(**kwargs) + + assert allowed in str(exc.value), ( + f"the error should list the accepted values {allowed}: {exc.value}" + ) + + def test_dates_stay_strings(self): + """Test that a date is kept as text rather than coerced to a date object. + + Test scenario: + `Catchment.__init__` parses the dates itself with `fmt`. If the schema coerced + them, `strptime` would be handed a `date` and raise, which is exactly why the + YAML quotes them. + """ + config = CatchmentConfig(name="Coello", start="2009-01-01", end="2009-01-10") + + assert isinstance(config.start, str), ( + f"start should be str, got {type(config.start)}" + ) + assert config.start == "2009-01-01", f"start was altered: {config.start}" + + +class TestMeteoConfig: + """Tests for the `meteo` block.""" + + def test_raster_defaults_match_the_reader(self): + """Test that the raster-reading defaults are the ones `from_rasters` uses. + + Test scenario: + A configuration that names three folders and nothing else must read them the way + the loader would by default, or the YAML would silently change the date parsing. + """ + config = MeteoConfig() + + assert config.source == "rasters", f"expected 'rasters', got {config.source}" + assert config.glob == "*.tif", f"unexpected glob default: {config.glob}" + assert config.regex_string == r"\d{4}.\d{2}.\d{2}", ( + f"unexpected regex default: {config.regex_string}" + ) + assert config.file_name_data_fmt is None, ( + "the date format should be inferred by default" + ) + + def test_an_unknown_source_is_refused(self): + """Test that `source` accepts only the three loaders that exist. + + Test scenario: + `source` selects which `MeteoInputs` constructor runs, so an unknown value has no + loader to dispatch to and must fail here rather than fall through. + """ + with pytest.raises(ValidationError, match="Input should be") as exc: + MeteoConfig(source="zarr") + + assert "'rasters', 'netcdf' or 'netcdf_files'" in str(exc.value), ( + f"the error should name the three loaders: {exc.value}" + ) + + +class TestConceptualModelConfig: + """Tests for the `conceptual_model` block.""" + + def test_initial_condition_must_hold_five_states(self): + """Test that the state vector is required to be exactly five long. + + Test scenario: + HBV indexes `[sp, sm, uz, lz, wc]` by position, so a shorter vector does not + raise where it is built -- it reads the wrong state, or runs off the end deep in + the per-cell loop. + """ + with pytest.raises(ValidationError, match="at least 5 items") as exc: + ConceptualModelConfig( + model_class="HBVBergestrom92", + catchment_area=1530, + initial_condition=[0, 5, 5, 5], + ) + + assert "initial_condition" in str(exc.value), ( + f"the error should name the field: {exc.value}" + ) + + def test_catchment_area_must_be_positive(self): + """Test that a non-positive catchment area is refused. + + Test scenario: + The area scales depth to discharge, so zero or a negative value produces a + hydrograph that is silently zero or sign-flipped rather than an error. + """ + with pytest.raises(ValidationError, match="greater than 0"): + ConceptualModelConfig( + model_class="HBVBergestrom92", + catchment_area=-5, + initial_condition=[0, 5, 5, 5, 0], + ) + + def test_model_class_keeps_its_name(self): + """Test that the `model_class` field survives pydantic's protected namespace. + + Test scenario: + Pydantic reserves the `model_` prefix. The YAML key is `model_class`, so the + namespace is cleared rather than the field renamed -- this pins that it stayed. + """ + config = ConceptualModelConfig( + model_class="HBVBergestrom92", + catchment_area=1530, + initial_condition=[0, 5, 5, 5, 0], + ) + + assert config.model_class == "HBVBergestrom92", ( + f"model_class did not round-trip: {config.model_class}" + ) + assert config.q_init is None, "q_init should default to None" + + +class TestStrictKeys: + """Tests for the `extra=forbid` setting shared by every block.""" + + @pytest.mark.parametrize( + "model, kwargs", + [ + ( + CatchmentConfig, + {"name": "c", "start": "2009-01-01", "end": "2009-01-10"}, + ), + (MeteoConfig, {}), + (FlowNetworkConfig, {"flow_accumulation": "acc.tif"}), + (ParametersConfig, {"path": "p"}), + (GaugesConfig, {"discharge": "q.csv"}), + (OutputsConfig, {}), + ], + ids=["catchment", "meteo", "flow_network", "parameters", "gauges", "outputs"], + ) + def test_an_unknown_key_is_refused(self, model, kwargs): + """Test that every block rejects a key it does not define. + + Args: + model: The block under test. + kwargs: The minimum valid arguments for it. + + Test scenario: + A misspelled key would otherwise be dropped silently, and the input it was meant + to supply would go missing far from the typo that caused it. + """ + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + model(**{**kwargs, "definitely_not_a_field": 1}) + + +class TestRunConfigCrossFieldRules: + """Tests for the rules tying the blocks to `catchment.spatial_resolution`.""" + + def test_a_complete_distributed_configuration_validates(self, distributed_mapping): + """Test that the fixture itself is accepted, so the negative cases mean something. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + Every rejection test below mutates one field of this mapping, so it has to be + valid to begin with or those tests would pass for the wrong reason. + """ + config = RunConfig.model_validate(distributed_mapping) + + assert config.catchment.spatial_resolution == "distributed" + assert config.flow_network is not None, "the flow network should be parsed" + assert config.gauges is not None, "the gauges block should be parsed" + + def test_a_complete_lumped_configuration_validates(self, lumped_mapping): + """Test that a lumped configuration with no grid blocks is accepted. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + The lumped shape is the other half of the schema: one CSV of averaged drivers, + no flow network, and no gauge table. + """ + config = RunConfig.model_validate(lumped_mapping) + + assert config.flow_network is None, "a lumped run carries no flow network" + assert config.meteo.path == lumped_mapping["meteo"]["path"], ( + f"the lumped meteo CSV was not kept: {config.meteo.path}" + ) + assert config.gauges.table is None, "a lumped run needs no gauge table" + + def test_distributed_requires_a_flow_network(self, distributed_mapping): + """Test that a distributed run without a routing network is refused. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + The network supplies the grid every cube is checked against, so without it the + run fails later on a shape mismatch that says nothing about the missing block. + """ + del distributed_mapping["flow_network"] + + with pytest.raises(ValidationError, match="needs a flow_network block") as exc: + RunConfig.model_validate(distributed_mapping) + + assert "distributed" in str(exc.value), ( + f"the error should name the resolution that requires it: {exc.value}" + ) + + def test_distributed_requires_a_gauge_table_when_gauges_are_given( + self, distributed_mapping + ): + """Test that a distributed `gauges` block without a table is refused. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + The table is what locates each gauge on the grid. Supplying discharge without it + is a half-configured block, distinct from omitting gauges altogether. + """ + del distributed_mapping["gauges"]["table"] + + with pytest.raises(ValidationError, match="needs gauges.table"): + RunConfig.model_validate(distributed_mapping) + + @pytest.mark.parametrize( + "driver", ["precipitation", "temperature", "evapotranspiration"] + ) + def test_distributed_requires_all_three_drivers(self, distributed_mapping, driver): + """Test that each missing driver is reported by name. + + Args: + distributed_mapping: A complete distributed configuration. + driver: The driver removed from the meteo block. + + Test scenario: + The conceptual model reads all three every step, so a configuration missing one + cannot run -- and the error has to say which, since the three are interchangeable + in shape. + """ + del distributed_mapping["meteo"][driver] + + with pytest.raises(ValidationError, match="missing") as exc: + RunConfig.model_validate(distributed_mapping) + + assert driver in str(exc.value), ( + f"the error should name the missing driver {driver}: {exc.value}" + ) + + def test_netcdf_source_requires_a_path(self, distributed_mapping): + """Test that `source: netcdf` without `meteo.path` is refused. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + For this source the three driver fields are variable *names* inside one file, so + without the file there is nothing to read them from. + """ + del distributed_mapping["meteo"]["path"] + + with pytest.raises(ValidationError, match="needs meteo.path"): + RunConfig.model_validate(distributed_mapping) + + def test_lumped_requires_the_meteo_csv(self, lumped_mapping): + """Test that a lumped run without `meteo.path` is refused. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + Lumped mode has no grid to fall back on: `read_lumped_inputs` needs the single + CSV of catchment-average drivers, and nothing else supplies them. + """ + del lumped_mapping["meteo"]["path"] + + with pytest.raises(ValidationError, match="needs meteo.path"): + RunConfig.model_validate(lumped_mapping) + + def test_parameters_and_gauges_may_both_be_omitted(self, distributed_mapping): + """Test that a configuration carrying neither optional block still validates. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + A calibration derives its parameters from the bounds it is given rather than + reading a fitted set, and a run that is not scored against observations has no + gauges. Both blocks are therefore optional. + """ + del distributed_mapping["parameters"] + del distributed_mapping["gauges"] + + config = RunConfig.model_validate(distributed_mapping) + + assert config.parameters is None, "parameters should be absent, not defaulted" + assert config.gauges is None, "gauges should be absent, not defaulted" + + +class TestCatchmentFromYaml: + """Tests for `Catchment.from_yaml`, which turns a configuration into a built model.""" + + def test_a_distributed_configuration_populates_every_input( + self, distributed_mapping, tmp_path, coello_cat_area, coello_initial_cond + ): + """Test that the builder makes each `read_*` call the configuration asks for. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + coello_cat_area: Expected catchment area. + coello_initial_cond: Expected initial state. + + Test scenario: + This is the whole point of the alternate constructor: the attributes a + hand-written script assembled by calling the readers in order must all be + populated by one call. + """ + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.name == "Coello", f"name not set: {model.name}" + assert model.spatial_resolution == "distributed" + assert model.meteo is not None, "meteo was not assigned" + assert model.flow_network is not None, "flow_network was not assigned" + assert model.parameters is not None, "parameters were not read" + assert model.lumped_model is not None, "the conceptual model was not read" + assert model.GaugesTable is not None, "the gauge table was not read" + assert model.QGauges is not None, "the discharge was not read" + assert model.area == coello_cat_area, f"area not set: {model.area}" + assert model.initial_cond == coello_initial_cond, ( + f"initial condition not set: {model.initial_cond}" + ) + + def test_the_drivers_cover_the_model_period(self, distributed_mapping, tmp_path): + """Test that the meteo window defaults to the catchment's own dates. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The drivers pair with the model's date index by position, so a file spanning a + longer record has to be trimmed to the run. Omitting `meteo.start` / `meteo.end` + should fall back to the catchment block rather than read the file whole. + """ + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo.time_steps == len(model.date_index), ( + f"drivers hold {model.meteo.time_steps} steps, model spans " + f"{len(model.date_index)}" + ) + assert model.meteo.time[0] == model.date_index[0], ( + f"drivers start at {model.meteo.time[0]}, model at {model.date_index[0]}" + ) + + def test_an_explicit_meteo_window_overrides_the_catchment_dates( + self, distributed_mapping, tmp_path + ): + """Test that `meteo.start` / `meteo.end` win when given. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The fallback is a convenience, not a rule: a caller who states the window + explicitly is asking for that slice of the record. + """ + distributed_mapping["meteo"]["start"] = "2009-01-03" + distributed_mapping["meteo"]["end"] = "2009-01-07" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo.time_steps == 5, ( + f"03 to 07 inclusive is five steps, got {model.meteo.time_steps}" + ) + + def test_omitting_parameters_leaves_them_unread( + self, distributed_mapping, tmp_path + ): + """Test that no parameter set is read when the block is absent. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The calibration shape. Reading a fitted set here would overwrite what the + optimiser is about to supply, so the builder must skip the call entirely. + """ + del distributed_mapping["parameters"] + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.parameters is None, ( + f"parameters should be unread, got {type(model.parameters)}" + ) + assert model.meteo is not None, "the rest of the build should still have run" + + def test_omitting_gauges_leaves_them_unread(self, distributed_mapping, tmp_path): + """Test that no gauge data is read when the block is absent. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + A run that is only inspected, never scored, carries no observations -- and the + gauge readers would otherwise fail on a path that was never configured. + """ + del distributed_mapping["gauges"] + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.GaugesTable is None, "no gauge table should have been read" + assert model.QGauges is None, "no discharge should have been read" + + def test_the_routing_label_is_the_literal_the_router_compares( + self, distributed_mapping, tmp_path + ): + """Test that lower-case `muskingum` reaches the model as `"Muskingum"`. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + `distrrm.SpatialRouting` tests `routing_method != "Muskingum"` case-sensitively, + and `Catchment.__init__` stores whatever it is given verbatim. A lower-case + spelling would send every cell down the MAXBAS branch and read `bankfull_depth`, + which is None outside the flood model. + """ + distributed_mapping["catchment"]["routing_method"] = "muskingum" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.routing_method == "Muskingum", ( + f"the router compares against 'Muskingum' exactly, got {model.routing_method!r}" + ) + + def test_an_unregistered_model_class_is_refused_by_name( + self, distributed_mapping, tmp_path + ): + """Test that a conceptual model the registry does not know is rejected. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The YAML names the model as a string, so the builder has to resolve it. An + unknown name must say what is available rather than fail on a None class later. + """ + distributed_mapping["conceptual_model"]["model_class"] = "HBV97" + + with pytest.raises(ValueError, match="not.*registered") as exc: + Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert "HBVBergestrom92" in str(exc.value), ( + f"the error should list the known models: {exc.value}" + ) + + @pytest.mark.parametrize("cls", [Catchment, Calibration]) + def test_the_builder_returns_the_class_it_was_called_on( + self, distributed_mapping, tmp_path, cls + ): + """Test that a subclass taking the same constructor arguments builds its own type. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + cls: The class the classmethod is called on. + + Test scenario: + `Calibration` extends `Catchment` with the same constructor signature, so + `Calibration.from_yaml(...)` should hand back a `Calibration` the calibration + methods can be called on, not a bare `Catchment`. + """ + model = cls.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert isinstance(model, cls), ( + f"expected a {cls.__name__}, got {type(model).__name__}" + ) + + def test_run_cannot_be_built_because_it_takes_no_constructor_arguments( + self, distributed_mapping, tmp_path + ): + """Test that `Run.from_yaml` fails loudly rather than building something unusable. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + `Run` inherits the classmethod but overrides `__init__` to take only `self`, and + its entry points are called unbound on a catchment (`Run.RunHapi(model)`). Pins + that the mismatch surfaces as a `TypeError` at the constructor rather than as a + half-built model, so the docstring's warning stays true. + """ + with pytest.raises(TypeError, match="unexpected keyword argument"): + Run.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + def test_a_lumped_configuration_reads_the_averaged_driver_csv( + self, lumped_mapping, tmp_path + ): + """Test that the lumped branch reads one CSV instead of a grid. + + Args: + lumped_mapping: A complete lumped configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Lumped mode is the other half of the builder: `read_lumped_inputs` fills `data` + from a single file of catchment-average drivers, and no `MeteoInputs` grid or + flow network is built at all. + """ + model = Catchment.from_yaml(write_yaml(lumped_mapping, tmp_path)) + + assert model.spatial_resolution == "lumped" + assert model.data is not None, "the averaged drivers were not read" + assert model.meteo is None, "a lumped run should build no driver grid" + assert model.flow_network is None, "a lumped run should build no flow network" + assert model.parameters is not None, "the lumped parameter file was not read" + + def test_a_lumped_run_reads_discharge_without_a_gauge_table( + self, lumped_mapping, tmp_path + ): + """Test that lumped gauges are read from one file, with no table lookup. + + Args: + lumped_mapping: A complete lumped configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The gauge table exists to locate gauges on a grid, which lumped mode has none + of, so the builder must skip `read_gauge_table` and still read the discharge. + """ + model = Catchment.from_yaml(write_yaml(lumped_mapping, tmp_path)) + + assert model.QGauges is not None, "the observed discharge was not read" + assert model.GaugesTable is None, "a lumped run should read no gauge table" + + def test_an_invalid_configuration_fails_before_anything_is_read( + self, distributed_mapping, tmp_path + ): + """Test that validation runs ahead of the first reader. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Reading rasters is the slow part of a build. A configuration that cannot work + should be rejected on the parsed mapping, not after minutes of I/O. + """ + distributed_mapping["catchment"]["spatial_resolution"] = "semi" + + with pytest.raises(ValidationError, match="Input should be"): + Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + def test_the_configuration_is_read_from_the_given_path( + self, distributed_mapping, tmp_path + ): + """Test that two different files build two differently-named models. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Guards against the path being ignored in favour of something cached or + hard-coded -- the name has to come from the file that was named. + """ + first = write_yaml(distributed_mapping, tmp_path) + renamed = copy.deepcopy(distributed_mapping) + renamed["catchment"]["name"] = "Elsewhere" + second = tmp_path / "other.yaml" + second.write_text(yaml.safe_dump(renamed), encoding="utf-8") + + assert Catchment.from_yaml(first).name == "Coello" + assert Catchment.from_yaml(str(second)).name == "Elsewhere" From 742e7ede3f8fb371daa27480aa94bb183eb75297 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:25:26 +0200 Subject: [PATCH 13/61] fix(config): require a flow-direction raster for a Muskingum run `flow_direction` is optional on the block because MAXBAS sends every cell straight to the outlet and never reads one. Nothing tied that to the routing method, so a distributed Muskingum config without it validated, built, and then died inside `Run.RunHapi` on `flow_dir_arr.shape` -- a bare `AttributeError` naming neither the block nor the file, raised after the meteo cubes, the accumulation raster and the whole parameter folder had been read. `muskingum` is the default, so the likeliest way in is copying the shipped MAXBAS example's `flow_network` block, which legitimately omits the raster. The validator now states the rule, with a test on each side of it. --- src/hapi/config.py | 12 ++++++++++++ tests/test_config.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/hapi/config.py b/src/hapi/config.py index ca78ef0d..2a120ca8 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -283,6 +283,18 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: "catchment.spatial_resolution is 'distributed', which needs a flow_network " "block" ) + # `flow_direction` is optional on the block because MAXBAS sends every cell straight + # to the outlet and never reads one. Muskingum routes along the network, so without + # it the build succeeds and `Run.RunHapi` dereferences a None array after every + # raster has been read. + if ( + self.catchment.routing_method == "muskingum" + and self.flow_network.flow_direction is None + ): + raise ValueError( + "catchment.routing_method is 'muskingum', which routes along the network, " + "so flow_network.flow_direction is required" + ) # Only when gauges are configured at all: a distributed run that is not scored # against observations omits the block entirely. if self.gauges is not None and self.gauges.table is None: diff --git a/tests/test_config.py b/tests/test_config.py index 1eff65df..a064fb3b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -384,6 +384,44 @@ def test_distributed_requires_a_flow_network(self, distributed_mapping): f"the error should name the resolution that requires it: {exc.value}" ) + def test_muskingum_requires_a_flow_direction_raster(self, distributed_mapping): + """Test that a Muskingum run without a direction raster is refused at parse time. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + `flow_direction` is optional on the block because MAXBAS never reads one. Muskingum + does, and it is the default routing method -- so a config copied from the MAXBAS + example builds fine and then dereferences a None array inside the routing loop, + after every raster has been read. The rule has to be stated here instead. + """ + del distributed_mapping["flow_network"]["flow_direction"] + + with pytest.raises(ValidationError, match="flow_network.flow_direction is required"): + RunConfig.model_validate(distributed_mapping) + + def test_maxbas_does_not_require_a_flow_direction_raster(self, distributed_mapping): + """Test that the triangular path still accepts a network without a direction raster. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + The counterpart to the rule above: MAXBAS routes every cell straight to the outlet, + so requiring the raster there would reject the configuration the shipped MAXBAS + example uses. + """ + distributed_mapping["catchment"]["routing_method"] = "maxbas" + distributed_mapping["parameters"]["maxbas"] = True + del distributed_mapping["flow_network"]["flow_direction"] + + config = RunConfig.model_validate(distributed_mapping) + + assert config.flow_network.flow_direction is None, ( + "MAXBAS should be allowed to omit the direction raster" + ) + def test_distributed_requires_a_gauge_table_when_gauges_are_given( self, distributed_mapping ): From cec4bcc88a4c72062481a4af30d020cbd1434965 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:27:05 +0200 Subject: [PATCH 14/61] fix(catchment): decode the run configuration as UTF-8 `Path(path).read_text()` uses the locale codec, so a configuration carrying a non-ASCII catchment name or path decoded to different text on a machine whose default is not UTF-8 -- silently, since the mojibaked result is still valid YAML. The name reaches result filenames and plot titles; a corrupted path produces a FileNotFoundError naming something the user cannot find in their own file. Version-dependent, so it passed on 3.14 and failed on the 3.11 floor the project declares and CI runs. The new test is green on both. --- src/hapi/catchment.py | 6 +++++- tests/test_config.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 11853f55..042eb6d4 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -274,7 +274,11 @@ def from_yaml(cls, path: str) -> Self: ``` """ - config = RunConfig.model_validate(yaml.safe_load(Path(path).read_text())) + # Explicit encoding: without it the file is decoded with the locale codec, so a + # non-ASCII catchment name or path mojibakes on a machine whose default is not UTF-8 + # -- and does so silently, since the corrupted text is still valid YAML. + text = Path(path).read_text(encoding="utf-8") + config = RunConfig.model_validate(yaml.safe_load(text)) catchment = config.catchment model = cls( diff --git a/tests/test_config.py b/tests/test_config.py index a064fb3b..08c33310 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -756,6 +756,29 @@ def test_a_lumped_run_reads_discharge_without_a_gauge_table( assert model.QGauges is not None, "the observed discharge was not read" assert model.GaugesTable is None, "a lumped run should read no gauge table" + def test_a_non_ascii_name_survives_the_read(self, distributed_mapping, tmp_path): + """Test that the configuration is decoded as UTF-8 whatever the platform default is. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Without an explicit encoding the file is decoded with the locale codec, so a + non-ASCII name mojibakes on a machine that does not default to UTF-8 -- silently, + because the corrupted text is still valid YAML. The name reaches result filenames + and plot titles, and the same risk applies to every path field. + """ + distributed_mapping["catchment"]["name"] = "Río Coello" + path = tmp_path / "config.yaml" + path.write_text( + yaml.safe_dump(distributed_mapping, allow_unicode=True), encoding="utf-8" + ) + + model = Catchment.from_yaml(str(path)) + + assert model.name == "Río Coello", f"the name was corrupted on read: {model.name!r}" + def test_an_invalid_configuration_fails_before_anything_is_read( self, distributed_mapping, tmp_path ): From 8d5cadb8668ce870cca70d15f864930baf27f3e9 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:29:47 +0200 Subject: [PATCH 15/61] fix(catchment): canonicalise routing_method in the constructor `__init__` case-folded `spatial_resolution` and `temporal_resolution` but stored `routing_method` verbatim, while `distrrm.SpatialRouting` compares `routing_method != "Muskingum"` case-sensitively and its false branch reads `bankfull_depth` -- None outside the flood model. A lower-case "muskingum" therefore routed every cell down the MAXBAS branch and raised `TypeError: 'NoneType' object is not subscriptable`. The branch had been papering over this in `from_yaml` alone, which left the bug live on the hand-written path every example and downstream script uses, and made `Catchment.routing_method` mean different things depending on how the object was built. The constructor now validates and canonicalises like the other two enumerated arguments, and `_ROUTING_METHOD_LABELS` is gone. "Kinematic" is in the accepted set: that same `!= "Muskingum"` comparison is how `Run.RunFloodModel` selects its path, so rejecting it would have broken the flood model. --- src/hapi/catchment.py | 39 ++++++++++++++++++-------- tests/test_config.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 12 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 042eb6d4..79475678 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -68,14 +68,16 @@ "HBV": HBV, } -#: `__init__` stores `routing_method` verbatim, with none of the case-folding it applies to -#: `spatial_resolution` / `temporal_resolution`. `distrrm.SpatialRouting` -- the Muskingum loop -#: `Run.RunHapi` reaches -- then tests `Model.routing_method != "Muskingum"`, an exact, -#: case-sensitive match. Any other spelling sends every cell down the MAXBAS branch, which reads -#: `bankfull_depth`: None outside the flood model, so a `TypeError`. The YAML vocabulary is -#: lower case, so it is translated here to the literal the internals expect. MAXBAS never -#: reaches that comparison, so its label is cosmetic. -_ROUTING_METHOD_LABELS = {"muskingum": "Muskingum", "maxbas": "MAXBAS"} +#: Accepted routing methods, mapped to the one spelling the internals compare against. +#: `distrrm.SpatialRouting` tests `routing_method != "Muskingum"` exactly, so the constructor +#: canonicalises rather than storing what it was handed. `"Kinematic"` belongs here because +#: that comparison is also how the flood model selects its own path: a non-Muskingum method +#: with a real `bankfull_depth` skips the cell, which `Run.RunFloodModel` relies on. +ROUTING_METHODS = { + "muskingum": "Muskingum", + "maxbas": "MAXBAS", + "kinematic": "Kinematic", +} @contextmanager @@ -134,14 +136,17 @@ def __init__( "Distributed". Default is "Lumped". temporal_resolution (str, optional): "Hourly" or "Daily". Default is "Daily". - routing_method (str, optional): Routing method name. - Default is "Muskingum". + routing_method (str, optional): "Muskingum", "MAXBAS" or + "Kinematic", matched case-insensitively and stored + canonicalised. Default is "Muskingum". Raises: ValueError: If `spatial_resolution` is not "lumped" or "distributed". ValueError: If `temporal_resolution` is not "daily" or "hourly". + ValueError: If `routing_method` is not "Muskingum", "MAXBAS" or + "Kinematic". """ self.name = name self.start = dt.datetime.strptime(start_data, fmt) @@ -170,7 +175,17 @@ def __init__( self.conversion_factor = CONVERSION_FACTOR * 1 / 24 self.date_index = pd.date_range(self.start, self.end, freq="h") - self.routing_method = routing_method + # Canonicalised like the two resolutions above, and for a sharper reason: + # `distrrm.SpatialRouting` tests `routing_method != "Muskingum"` case-sensitively, and + # the false branch reads `bankfull_depth`, which is None outside the flood model. Left + # verbatim, a lower-case "muskingum" therefore routed every cell down the MAXBAS branch + # and raised `TypeError: 'NoneType' object is not subscriptable`. + if routing_method.lower() not in ROUTING_METHODS: + raise ValueError( + f"available routing methods are {', '.join(map(repr, ROUTING_METHODS))}, " + f"got {routing_method!r}" + ) + self.routing_method = ROUTING_METHODS[routing_method.lower()] self.parameters: np.ndarray | list | None = None self.data: np.ndarray | None = None #: The three meteorological drivers. Assign a :class:`~hapi.inputs.MeteoInputs` @@ -288,7 +303,7 @@ def from_yaml(cls, path: str) -> Self: fmt=catchment.fmt, spatial_resolution=catchment.spatial_resolution, temporal_resolution=catchment.temporal_resolution, - routing_method=_ROUTING_METHOD_LABELS[catchment.routing_method], + routing_method=catchment.routing_method, ) distributed = catchment.spatial_resolution == "distributed" diff --git a/tests/test_config.py b/tests/test_config.py index 08c33310..51c6d74f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -513,6 +513,70 @@ def test_parameters_and_gauges_may_both_be_omitted(self, distributed_mapping): assert config.gauges is None, "gauges should be absent, not defaulted" +class TestRoutingMethodNormalisation: + """Tests for the `routing_method` canonicalisation in `Catchment.__init__`.""" + + @pytest.mark.parametrize( + "given, stored", + [ + ("muskingum", "Muskingum"), + ("Muskingum", "Muskingum"), + ("MUSKINGUM", "Muskingum"), + ("maxbas", "MAXBAS"), + ("MAXBAS", "MAXBAS"), + ("kinematic", "Kinematic"), + ], + ) + def test_any_casing_is_stored_canonically(self, given, stored): + """Test that the constructor stores one spelling whatever casing it is handed. + + Args: + given: The spelling passed to the constructor. + stored: The canonical spelling expected on the model. + + Test scenario: + `distrrm.SpatialRouting` compares `routing_method != "Muskingum"` exactly, and its + false branch reads `bankfull_depth`, which is None outside the flood model. A + lower-case "muskingum" stored verbatim therefore routed every cell down the MAXBAS + branch and raised `TypeError: 'NoneType' object is not subscriptable`. + """ + model = Catchment( + "coello", "2009-01-01", "2009-01-10", routing_method=given + ) + + assert model.routing_method == stored, ( + f"{given!r} should be stored as {stored!r}, got {model.routing_method!r}" + ) + + def test_kinematic_is_accepted_for_the_flood_model(self): + """Test that the flood model's routing method is still a legal value. + + Test scenario: + `Run.RunFloodModel` relies on the same `!= "Muskingum"` comparison to skip cells + with a real `bankfull_depth`, so "Kinematic" is a working value and must not be + rejected by the new validation. + """ + model = Catchment( + "coello", "2009-01-01", "2009-01-10", routing_method="Kinematic" + ) + + assert model.routing_method == "Kinematic" + + def test_an_unknown_routing_method_is_refused(self): + """Test that an unrecognised routing method fails at construction. + + Test scenario: + Any unknown spelling silently selects the non-Muskingum branch downstream, so it + has to be caught here rather than surface as a `TypeError` on `bankfull_depth`. + """ + with pytest.raises(ValueError, match="available routing methods") as exc: + Catchment("coello", "2009-01-01", "2009-01-10", routing_method="diffusive") + + assert "diffusive" in str(exc.value), ( + f"the error should echo the value given: {exc.value}" + ) + + class TestCatchmentFromYaml: """Tests for `Catchment.from_yaml`, which turns a configuration into a built model.""" From 9b6e8d78f03169a885e811bc6d072a0195d09298 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:33:36 +0200 Subject: [PATCH 16/61] fix(inputs): parse each meteo bound with the format it was written in When the `meteo` block states no window it inherits the catchment's dates, which are written in `catchment.fmt` -- but `from_config` parsed them with `meteo.fmt`. The two are independent fields with independent defaults. The loud case merely confuses, blaming a date the user never wrote that way. The quiet one is worse: between two mutually parseable layouts such as "%d-%m-%Y" and "%m-%d-%Y" the drivers are windowed to the wrong period, and since `MeteoInputs` pairs with `date_index` by position the model then runs on drivers offset from its own calendar with no error at all. `from_config` now takes the inherited format alongside the inherited dates, resolves each bound with the format it belongs to, and passes datetimes on so no loader can re-parse them. The bound annotations widen to match `_as_datetime`, which has always accepted a datetime; `read_rasters` rejects one explicitly on the `date=False` path, where bounds are indices rather than dates and `int()` would have failed several frames down. --- src/hapi/catchment.py | 5 +++- src/hapi/inputs.py | 54 +++++++++++++++++++++++++++++++----------- tests/test_config.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 79475678..95bd0683 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -309,7 +309,10 @@ def from_yaml(cls, path: str) -> Self: distributed = catchment.spatial_resolution == "distributed" if distributed: model.meteo = MeteoInputs.from_config( - config.meteo, start=catchment.start, end=catchment.end + config.meteo, + start=catchment.start, + end=catchment.end, + fmt=catchment.fmt, ) model.flow_network = FlowNetwork.from_rasters( config.flow_network.flow_accumulation, diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index ec4c99ea..20cd5087 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -156,8 +156,8 @@ def read_rasters( regex_string: str = r"\d{4}.\d{2}.\d{2}", date: bool = True, file_name_data_fmt: str | None = None, - start: str | int | None = None, - end: str | int | None = None, + start: str | int | dt.datetime | None = None, + end: str | int | dt.datetime | None = None, fmt: str = "%Y-%m-%d", gdal_env: dict[str, str] | None = None, ) -> Datacube: @@ -231,6 +231,13 @@ def read_rasters( ) if not date: + # The numeric ordering bounds by the index in the file name, so a datetime has no + # meaning here -- `int()` would fail on it several frames down. + if isinstance(start, dt.datetime) or isinstance(end, dt.datetime): + raise TypeError( + "a datetime bound needs date=True; with date=False the rasters are ordered " + "by the number in their name, so start/end are indices" + ) return _read_by_index(path, glob, regex_string, start, end, gdal_env) return Datacube.from_files(path, glob=glob, gdal_env=gdal_env) @@ -950,8 +957,8 @@ def from_rasters( regex_string: str = r"\d{4}.\d{2}.\d{2}", date: bool = True, file_name_data_fmt: str | None = None, - start: str | int | None = None, - end: str | int | None = None, + start: str | int | dt.datetime | None = None, + end: str | int | dt.datetime | None = None, fmt: str = "%Y-%m-%d", gdal_env: dict[str, str] | None = None, ) -> MeteoInputs: @@ -1145,6 +1152,7 @@ def from_config( config: MeteoConfig, start: str | None = None, end: str | None = None, + fmt: str = "%Y-%m-%d", ) -> MeteoInputs: """Build the drivers with whichever loader the configuration's `source` names. @@ -1153,11 +1161,19 @@ def from_config( whose variables the block names. `hapi.config.RunConfig` has already checked that the fields the chosen source needs are set, so this calls the loader directly. + Each bound is parsed with the format it was written in -- `config.fmt` for a bound the + block states, `fmt` for one inherited from the caller -- and handed on as a `datetime`. + The two formats are independent fields, so parsing an inherited bound with the block's + format would either fail loudly or, between two mutually parseable layouts such as + `"%d-%m-%Y"` and `"%m-%d-%Y"`, silently window the drivers to the wrong period. + Args: config: The `meteo` block of a distributed configuration. start: Window start used when `config.start` is unset, so the drivers can default to the period the model spans. `None` leaves the lower bound open. end: Window end used when `config.end` is unset. `None` leaves it open. + fmt: `strptime` format of `start` / `end`, the inherited bounds. Bounds stated by + the block are parsed with `config.fmt` instead. Returns: MeteoInputs: The three cubes plus the calendar, windowed to the requested period. @@ -1214,8 +1230,18 @@ def from_config( from_netcdf: The loader `source="netcdf"` dispatches to. from_netcdf_files: The loader `source="netcdf_files"` dispatches to. """ - start = config.start or start - end = config.end or end + # Resolve each bound with the format it was written in, then pass datetimes on so the + # loader's own `fmt` cannot re-parse them. + window_start = ( + _as_datetime(config.start, config.fmt) + if config.start is not None + else _as_datetime(start, fmt) + ) + window_end = ( + _as_datetime(config.end, config.fmt) + if config.end is not None + else _as_datetime(end, fmt) + ) # The three are optional on the model because a lumped configuration sets none of them, # while every distributed source needs all three. `RunConfig` enforces that, so reaching @@ -1246,8 +1272,8 @@ def from_config( glob=config.glob, regex_string=config.regex_string, file_name_data_fmt=config.file_name_data_fmt, - start=start, - end=end, + start=window_start, + end=window_end, fmt=config.fmt, **extra, ) @@ -1263,8 +1289,8 @@ def from_config( precipitation=precipitation, temperature=temperature, evapotranspiration=evapotranspiration, - start=start, - end=end, + start=window_start, + end=window_end, fmt=config.fmt, ) @@ -1272,8 +1298,8 @@ def from_config( precipitation, temperature, evapotranspiration, - start=start, - end=end, + start=window_start, + end=window_end, fmt=config.fmt, ) @@ -1286,8 +1312,8 @@ def raster_folder_to_netcdf( regex_string: str = r"\d{4}.\d{2}.\d{2}", date: bool = True, file_name_data_fmt: str | None = None, - start: str | int | None = None, - end: str | int | None = None, + start: str | int | dt.datetime | None = None, + end: str | int | dt.datetime | None = None, fmt: str = "%Y-%m-%d", gdal_env: dict[str, str] | None = None, ) -> Path: diff --git a/tests/test_config.py b/tests/test_config.py index 51c6d74f..d96e59b1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -820,6 +820,61 @@ def test_a_lumped_run_reads_discharge_without_a_gauge_table( assert model.QGauges is not None, "the observed discharge was not read" assert model.GaugesTable is None, "a lumped run should read no gauge table" + def test_the_inherited_window_is_parsed_with_the_catchment_format( + self, distributed_mapping, tmp_path + ): + """Test that fallback dates are read with `catchment.fmt`, not `meteo.fmt`. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + `catchment.fmt` and `meteo.fmt` are independent fields. When the meteo block + states no window it inherits the catchment's dates, which are written in the + catchment's format -- so parsing them with the meteo format either fails on a + date the user never wrote that way, or, between two mutually parseable layouts, + silently windows the drivers to the wrong period. + """ + distributed_mapping["catchment"]["fmt"] = "%d/%m/%Y" + distributed_mapping["catchment"]["start"] = "01/01/2009" + distributed_mapping["catchment"]["end"] = "10/01/2009" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo.time_steps == len(model.date_index), ( + f"drivers hold {model.meteo.time_steps} steps, model spans " + f"{len(model.date_index)}" + ) + assert model.meteo.time[0] == model.date_index[0], ( + f"window start {model.meteo.time[0]} does not match the model's " + f"{model.date_index[0]}" + ) + + def test_a_stated_meteo_window_uses_the_meteo_format( + self, distributed_mapping, tmp_path + ): + """Test that a window the block states is parsed with the block's own format. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The other half of the rule: `meteo.fmt` governs `meteo.start` / `meteo.end`, so a + block may describe its window in a different layout from the catchment's without + either being reinterpreted. + """ + distributed_mapping["meteo"]["fmt"] = "%d/%m/%Y" + distributed_mapping["meteo"]["start"] = "03/01/2009" + distributed_mapping["meteo"]["end"] = "07/01/2009" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo.time_steps == 5, ( + f"03 to 07 January inclusive is five steps, got {model.meteo.time_steps}" + ) + def test_a_non_ascii_name_survives_the_read(self, distributed_mapping, tmp_path): """Test that the configuration is decoded as UTF-8 whatever the platform default is. From fc13fc6f5feba914abfe7e3392f88a421608c1d6 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:35:37 +0200 Subject: [PATCH 17/61] fix(config): require the routing method and the parameter set to agree A `routing_method: maxbas` run pointed at a 12-parameter Muskingum set validated, built and ran to completion, with `DistMaxbas2` reading the Muskingum X as the MAXBAS value -- a hydrograph that is quietly wrong, which is the worst failure mode for a modelling tool. The reverse pairing is symmetric: `Run.RunHapi` reads K and X out of an 11-parameter MAXBAS set. The parameter-count check cannot catch either. A MAXBAS set holds 11 parameters and a Muskingum set 12, and `parameters.maxbas` is what selects which count is expected, so a disagreeing pair still counts correctly. `RunConfig` now requires the two to agree whenever a `parameters` block is present. A calibration declares none -- its parameters come from the bounds -- so nothing there is constrained. The check is scoped to distributed runs: a lumped run picks its routing function at call time, not from `routing_method`. --- src/hapi/config.py | 21 +++++++++++++++++-- tests/test_config.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 2a120ca8..1a9800e8 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -176,8 +176,11 @@ class ParametersConfig(BaseModel): Attributes: path: Folder of parameter rasters (distributed) or a single file (lumped). snow: Whether the parameter set includes the snow routine (15 parameters against 10). - maxbas: Whether the set carries the triangular-routing parameter. Independent of - `catchment.routing_method` -- this describes the parameter set, not the run. + maxbas: Whether the set carries the triangular-routing parameter. It describes the + parameter set rather than the run, but `RunConfig` requires it to agree with + `catchment.routing_method`: the two counts differ (11 against 12) and `maxbas` + is what selects which is expected, so a disagreeing pair still passes the count + check and then reads the wrong parameter as the routing one. """ model_config = _STRICT @@ -314,6 +317,20 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: ) if self.meteo.source == "netcdf" and self.meteo.path is None: raise ValueError("meteo.source is 'netcdf', which needs meteo.path") + # The parameter-count check cannot catch a mismatch here: a MAXBAS set holds 11 + # parameters and a Muskingum set 12, and `parameters.maxbas` is what selects which + # count is expected -- so a set that disagrees with the routing method still counts + # correctly. The run then completes, reading the Muskingum X as the MAXBAS value + # (or K and X out of a MAXBAS set), and produces a hydrograph that is quietly wrong. + if self.parameters is not None: + wants_maxbas = self.catchment.routing_method == "maxbas" + if wants_maxbas != self.parameters.maxbas: + raise ValueError( + f"catchment.routing_method is " + f"{self.catchment.routing_method!r} but parameters.maxbas is " + f"{self.parameters.maxbas}; the parameter set and the routing method " + f"must agree, or the run reads the wrong parameter as the routing one" + ) elif self.meteo.path is None: raise ValueError( "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the CSV " diff --git a/tests/test_config.py b/tests/test_config.py index d96e59b1..0a61df32 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -422,6 +422,55 @@ def test_maxbas_does_not_require_a_flow_direction_raster(self, distributed_mappi "MAXBAS should be allowed to omit the direction raster" ) + @pytest.mark.parametrize( + "routing_method, maxbas", + [("maxbas", False), ("muskingum", True)], + ids=["maxbas-run-muskingum-set", "muskingum-run-maxbas-set"], + ) + def test_the_routing_method_and_the_parameter_set_must_agree( + self, distributed_mapping, routing_method, maxbas + ): + """Test that a routing method mismatched to its parameter set is refused. + + Args: + distributed_mapping: A complete distributed configuration. + routing_method: The routing method declared on the catchment. + maxbas: The flag declared on the parameter set, deliberately disagreeing. + + Test scenario: + The parameter-count check cannot catch this: a MAXBAS set holds 11 parameters and + a Muskingum set 12, and `parameters.maxbas` selects which count is expected, so a + disagreeing pair counts correctly. The run then completes and reads the Muskingum + X as the MAXBAS value -- a hydrograph that is quietly wrong, the worst failure + mode for a modelling tool. + """ + distributed_mapping["catchment"]["routing_method"] = routing_method + distributed_mapping["parameters"]["maxbas"] = maxbas + if routing_method == "maxbas": + del distributed_mapping["flow_network"]["flow_direction"] + + with pytest.raises(ValidationError, match="must agree"): + RunConfig.model_validate(distributed_mapping) + + def test_a_configuration_without_parameters_skips_the_routing_cross_check( + self, distributed_mapping + ): + """Test that the cross-check does not fire when no parameter set is configured. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + A calibration declares no `parameters` block -- its parameters come from the + bounds -- so there is nothing to disagree with the routing method, and requiring + agreement would reject every calibration configuration. + """ + del distributed_mapping["parameters"] + + config = RunConfig.model_validate(distributed_mapping) + + assert config.parameters is None + def test_distributed_requires_a_gauge_table_when_gauges_are_given( self, distributed_mapping ): From 03e2e4cb1ba0dfd2590b7650239d0ff17611417e Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:38:02 +0200 Subject: [PATCH 18/61] feat(catchment): keep the run configuration on the model it built `outputs.results_dir` was parsed, shipped in an example, and read by nothing: `from_yaml` discarded the `RunConfig` after building, so no caller could reach it even in principle. The example that set it then hardcoded the same path in Python two cells later, teaching that editing the field has an effect it did not have. `from_yaml` now leaves the validated configuration on `model.config`, so the blocks the build does not itself consume stay reachable. The netcdf example reads both its output directory and its flow-accumulation path from there instead of restating them -- the latter answering the wart its own comment admitted, since `FlowNetwork` keeps the arrays but not the source path. The example also writes beside the other example outputs rather than into the repository root, and states its driver/model checks with `raise` rather than `assert`, which a branch about asserts not surviving `python -O` should not have shipped. --- .../coello-distributed-model-run-netcdf.py | 31 +++++++------- .../coello-distributed-model-run-netcdf.yaml | 3 +- src/hapi/catchment.py | 8 ++++ tests/test_config.py | 42 +++++++++++++++++++ 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py index 40d98a7d..fe488a7e 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py @@ -30,15 +30,12 @@ print(f"model steps : {len(Coello.date_index)}") print(f"meteo period : {Coello.meteo.time[0]} -> {Coello.meteo.time[-1]}") print(f"model period : {Coello.date_index[0]} -> {Coello.date_index[-1]}") -assert Coello.meteo.time_steps == len(Coello.date_index), ( - "the drivers must hold exactly as many steps as the model spans" -) -assert Coello.meteo.time[0] == Coello.date_index[0], ( - "the drivers must start where the model does" -) -assert Coello.meteo.time[-1] == Coello.date_index[-1], ( - "the drivers must end where the model does" -) +if Coello.meteo.time_steps != len(Coello.date_index): + raise ValueError("the drivers must hold exactly as many steps as the model spans") +if Coello.meteo.time[0] != Coello.date_index[0]: + raise ValueError("the drivers must start where the model does") +if Coello.meteo.time[-1] != Coello.date_index[-1]: + raise ValueError("the drivers must end where the model does") # %% Run the model """ @@ -82,12 +79,16 @@ print(f"WB= {Coello.metrics.loc['WB', gauge_id]:.2f}") # %% Save the routed discharge to rasters, one per time step -# save_results re-reads the flow-accumulation raster for georeferencing; FlowNetwork keeps only -# the arrays, not the source path, so this repeats the path already given in the YAML. -FlowAccPath = "tests/rrm/data/coello/gis/acc4000.tif" -SaveTo = "results/saved rasters/" -Coello.save_results(flow_acc_path=FlowAccPath, result=1, path=SaveTo) -print(f"rasters written to : {SaveTo}") +# Both paths come from the configuration rather than being restated here: `save_results` +# re-reads the flow-accumulation raster for georeferencing (FlowNetwork keeps only the arrays, +# not the source path), and `outputs.results_dir` says where the rasters go. +save_to = Coello.config.outputs.results_dir +Coello.save_results( + flow_acc_path=Coello.config.flow_network.flow_accumulation, + result=1, + path=save_to, +) +print(f"rasters written to : {save_to}") # %% Plot the hydrograph at the outlet gauge (row position, not the gauge id) Coello.plot_hydrograph(Coello.start, Coello.end, Coello.GaugesTable.index[-1]) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml index 0c4d8c0a..2e1bc88e 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml @@ -33,5 +33,6 @@ gauges: column: id fmt: "%Y-%m-%d" +# Written beside the other example outputs rather than into the repository root. outputs: - results_dir: results/saved rasters/ + results_dir: examples/hydrological-model/data/distributed_model/results/ diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 95bd0683..acc4c53c 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -226,6 +226,11 @@ def __init__( self.qlz: np.ndarray | None = None self.Qsim: np.ndarray | None = None self.metrics: pd.DataFrame | None = None + #: The configuration this model was built from, when it came from + #: :meth:`from_yaml`; `None` for a model assembled by hand. Carries the blocks the + #: build itself does not consume, such as `outputs`, so a caller need not restate a + #: path the file already gives. + self.config: RunConfig | None = None @classmethod def from_yaml(cls, path: str) -> Self: @@ -357,6 +362,9 @@ def from_yaml(cls, path: str) -> Self: fmt=gauges.fmt, ) + # Kept so the blocks the build does not itself consume stay reachable -- `outputs` + # above all, which describes where results go rather than what the model reads. + model.config = config return model def read_flow_path_length(self, path: str): diff --git a/tests/test_config.py b/tests/test_config.py index 0a61df32..7b1ffbd5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -947,6 +947,48 @@ def test_a_non_ascii_name_survives_the_read(self, distributed_mapping, tmp_path) assert model.name == "Río Coello", f"the name was corrupted on read: {model.name!r}" + def test_the_configuration_stays_reachable_on_the_model( + self, distributed_mapping, tmp_path + ): + """Test that blocks the build does not consume survive on `model.config`. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + `outputs` describes where results go, so nothing in the build reads it. Discarding + the config would leave it parsed but unreachable, forcing a caller to restate in + Python a path the file already gives -- which is what the shipped example used to + do. + """ + distributed_mapping["outputs"] = {"results_dir": "somewhere/else/"} + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.config is not None, "the configuration should be kept on the model" + assert model.config.outputs.results_dir == "somewhere/else/", ( + f"outputs did not survive: {model.config.outputs}" + ) + assert model.config.flow_network.flow_accumulation is not None, ( + "the flow-accumulation path should stay reachable for save_results" + ) + + def test_a_hand_built_model_has_no_configuration(self, coello_start_date, coello_end_date): + """Test that `config` is None on a model that was not built from a file. + + Args: + coello_start_date: Simulation start date. + coello_end_date: Simulation end date. + + Test scenario: + The attribute has to be safe to check on any catchment, so a hand-assembled one + reports no configuration rather than raising `AttributeError`. + """ + model = Catchment("coello", coello_start_date, coello_end_date) + + assert model.config is None, f"expected no configuration, got {model.config}" + def test_an_invalid_configuration_fails_before_anything_is_read( self, distributed_mapping, tmp_path ): From c8fcfb2ae4496689d0ee0b3ad818cfcc570c557b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:40:55 +0200 Subject: [PATCH 19/61] docs: name the exception each converted check actually raises The assert-to-raise conversion split some checks across two exception types but left four `Raises:` sections describing the old single one. Since the point of the conversion is that callers can now catch these by type, those sections are the contract and were wrong: the calibration entry points list `ValueError` for optimization arguments that `_check_optimization_args` rejects with `TypeError`, and `Parameters.__init__` lists `TypeError` for a `lumped_par_pos` length mismatch that raises `ValueError`. --- src/hapi/calibration.py | 15 +++++++++------ src/hapi/rrm/parameters.py | 7 ++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/hapi/calibration.py b/src/hapi/calibration.py index c1e2dea6..3b53093a 100644 --- a/src/hapi/calibration.py +++ b/src/hapi/calibration.py @@ -241,8 +241,9 @@ def run_calibration( - res[1]: The optimal parameter set. Raises: - ValueError: If input dimensions are inconsistent or if - optimization arguments are not dictionaries. + ValueError: If input dimensions are inconsistent. + TypeError: If either bundle of optimization arguments is not a + dict. """ # input dimensions # [rows,cols] = self.FlowAcc.ReadAsArray().shape @@ -388,8 +389,9 @@ def FW1Calibration( - res[1]: The optimal parameter set. Raises: - ValueError: If input dimensions are inconsistent or if - optimization arguments are not dictionaries. + ValueError: If input dimensions are inconsistent. + TypeError: If either bundle of optimization arguments is not a + dict. """ # input dimensions # [rows,cols] = self.FlowAcc.ReadAsArray().shape @@ -524,8 +526,9 @@ def lumpedCalibration( Raises: ValueError: If `basic_inputs` is missing required keys - `"Route"` or `"RoutingFn"`, or if optimization - arguments are not dictionaries. + `"Route"` or `"RoutingFn"`. + TypeError: If either bundle of optimization arguments is not a + dict. """ # basic inputs # check if all inputs are included diff --git a/src/hapi/rrm/parameters.py b/src/hapi/rrm/parameters.py index ddce27bf..7739eb8c 100644 --- a/src/hapi/rrm/parameters.py +++ b/src/hapi/rrm/parameters.py @@ -87,9 +87,10 @@ def __init__( ValueError: If `function` is not one of the ints 1, 2, 3 or 4. A `bool`, a `float` such as `2.0`, and an unhashable value are all rejected rather than coerced or allowed to raise `TypeError`. - TypeError: If `no_parameters` is not an integer, if - `no_lumped_par` is not an integer, or if the length of - `lumped_par_pos` does not match `no_lumped_par`. + TypeError: If `no_parameters` or `no_lumped_par` is not an + integer. + ValueError: If the length of `lumped_par_pos` does not match + `no_lumped_par`. ValueError: If `lumped_par_pos` is not a list when `no_lumped_par` >= 1. From 65e9244e7706433b556c64398df696ead52fa83c Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:40:56 +0200 Subject: [PATCH 20/61] fix(rrm): keep the maxbas guard firing on NaN `assert maxbas >= 1` fired for NaN, since the comparison is false. Rewriting it as `if maxbas < 1` inverted that: NaN is not less than 1 either, so the guard stopped firing and execution reached `int(round(nan, 0))`, raising "cannot convert float NaN to integer" -- a message naming neither maxbas nor the routine it came from. A calibrated MAXBAS value can legitimately be NaN in a masked cell, so the guard is written as `not maxbas >= 1` at all four sites, which preserves the original semantics exactly. --- src/hapi/routing.py | 7 ++++++- src/hapi/rrm/hbv.py | 7 ++++++- src/hapi/rrm/hbv_bergestrom92.py | 7 ++++++- src/hapi/rrm/hbv_lake.py | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/hapi/routing.py b/src/hapi/routing.py index 9e3c7796..dd508747 100644 --- a/src/hapi/routing.py +++ b/src/hapi/routing.py @@ -216,7 +216,12 @@ def triangular_routing_2(q, maxbas=1): >>> q_routed = Routing.triangular_routing_2(q, maxbas=3) """ # input data validation - if maxbas < 1: + # `not maxbas >= 1` rather than `maxbas < 1`: both are false for NaN, but + # the original assert fired on it. Keeping the negated form means a NaN + # maxbas -- which a calibration can produce in a masked cell -- still fails + # here, naming the parameter, instead of surfacing as a bare + # "cannot convert float NaN to integer" further in. + if not maxbas >= 1: raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas diff --git a/src/hapi/rrm/hbv.py b/src/hapi/rrm/hbv.py index 1312dc5f..1325aecc 100644 --- a/src/hapi/rrm/hbv.py +++ b/src/hapi/rrm/hbv.py @@ -496,7 +496,12 @@ def routing(self, q, maxbas=1): >>> print(q_routed.round(4)) [0. 0. 2.5 4. 2.5 0.5] """ - if maxbas < 1: + # `not maxbas >= 1` rather than `maxbas < 1`: both are false for NaN, but + # the original assert fired on it. Keeping the negated form means a NaN + # maxbas -- which a calibration can produce in a masked cell -- still fails + # here, naming the parameter, instead of surfacing as a bare + # "cannot convert float NaN to integer" further in. + if not maxbas >= 1: raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas # maxbas = int(maxbas) diff --git a/src/hapi/rrm/hbv_bergestrom92.py b/src/hapi/rrm/hbv_bergestrom92.py index 3e6dffcf..63236ca3 100644 --- a/src/hapi/rrm/hbv_bergestrom92.py +++ b/src/hapi/rrm/hbv_bergestrom92.py @@ -433,7 +433,12 @@ def routing(self, q, maxbas=1): >>> len(q_r) == len(q) True """ - if maxbas < 1: + # `not maxbas >= 1` rather than `maxbas < 1`: both are false for NaN, but + # the original assert fired on it. Keeping the negated form means a NaN + # maxbas -- which a calibration can produce in a masked cell -- still fails + # here, naming the parameter, instead of surfacing as a bare + # "cannot convert float NaN to integer" further in. + if not maxbas >= 1: raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas maxbas = int(round(maxbas, 0)) diff --git a/src/hapi/rrm/hbv_lake.py b/src/hapi/rrm/hbv_lake.py index cefae98d..ac66e3c6 100644 --- a/src/hapi/rrm/hbv_lake.py +++ b/src/hapi/rrm/hbv_lake.py @@ -493,7 +493,12 @@ def _routing(self, q, maxbas=1): >>> len(q_r) == len(q) True """ - if maxbas < 1: + # `not maxbas >= 1` rather than `maxbas < 1`: both are false for NaN, but + # the original assert fired on it. Keeping the negated form means a NaN + # maxbas -- which a calibration can produce in a masked cell -- still fails + # here, naming the parameter, instead of surfacing as a bare + # "cannot convert float NaN to integer" further in. + if not maxbas >= 1: raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") # Get integer part of maxbas maxbas = int(round(maxbas, 0)) From e93e75a1b9bf72da33e181cec49bfe03c6206073 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:43:34 +0200 Subject: [PATCH 21/61] fix(config): validate the dates, and reject blocks a lumped run cannot use Two gaps in the promise that a validated configuration is consumable without re-checking. Dates were never checked against the format they are written in, so `start: not-a-date` validated and failed later inside `Catchment.__init__`; nothing checked that a period runs forwards either, so a reversed one produced an empty date index and failed downstream on a shape mismatch. Both are now caught per field, named, and the period order with them. A lumped configuration could also carry a `flow_network` block or a grid `meteo.source`, both silently discarded -- the block simply never took effect, and a `source: netcdf` lumped run would hand a `.nc` path to `pd.read_csv`. `extra="forbid"` is used precisely so a misspelled key fails rather than being dropped, so accepting a correctly spelled but inapplicable one was the same silence by another route. --- src/hapi/config.py | 63 +++++++++++++++++++++++++--- tests/test_config.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 1a9800e8..7d6f0cfd 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -81,6 +81,7 @@ from __future__ import annotations +from datetime import datetime from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -331,9 +332,61 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: f"{self.parameters.maxbas}; the parameter set and the routing method " f"must agree, or the run reads the wrong parameter as the routing one" ) - elif self.meteo.path is None: - raise ValueError( - "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the CSV " - "of catchment-average drivers" - ) + else: + if self.meteo.path is None: + raise ValueError( + "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the " + "CSV of catchment-average drivers" + ) + # `extra="forbid"` exists so a misspelled key fails rather than being dropped; + # accepting a correctly spelled but inapplicable block would be the same silence + # by another route. A lumped run has no grid, so neither block can be honoured. + if self.flow_network is not None: + raise ValueError( + "catchment.spatial_resolution is 'lumped', which has no grid, so a " + "flow_network block cannot be used" + ) + if self.meteo.source != "rasters": + raise ValueError( + f"catchment.spatial_resolution is 'lumped', which reads meteo.path as a " + f"CSV of catchment-average drivers; meteo.source " + f"{self.meteo.source!r} does not apply" + ) + return self + + @model_validator(mode="after") + def _check_the_dates_parse_and_are_ordered(self) -> RunConfig: + """Check every date against the format it is written in, and that the period runs. + + Returns: + RunConfig: This config, unchanged. + + Raises: + ValueError: A date does not match its format, or a period ends before it starts. + """ + for label, value, fmt in ( + ("catchment.start", self.catchment.start, self.catchment.fmt), + ("catchment.end", self.catchment.end, self.catchment.fmt), + ("meteo.start", self.meteo.start, self.meteo.fmt), + ("meteo.end", self.meteo.end, self.meteo.fmt), + ): + if value is None: + continue + try: + datetime.strptime(value, fmt) + except ValueError as error: + raise ValueError( + f"{label} {value!r} does not match its format {fmt!r}: {error}" + ) from error + + for first, second, fmt, block in ( + (self.catchment.start, self.catchment.end, self.catchment.fmt, "catchment"), + (self.meteo.start, self.meteo.end, self.meteo.fmt, "meteo"), + ): + if first is None or second is None: + continue + if datetime.strptime(first, fmt) > datetime.strptime(second, fmt): + raise ValueError( + f"{block}.start {first!r} is after {block}.end {second!r}" + ) return self diff --git a/tests/test_config.py b/tests/test_config.py index 7b1ffbd5..d93e301e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -542,6 +542,81 @@ def test_lumped_requires_the_meteo_csv(self, lumped_mapping): with pytest.raises(ValidationError, match="needs meteo.path"): RunConfig.model_validate(lumped_mapping) + @pytest.mark.parametrize( + "block, field", + [("catchment", "start"), ("catchment", "end"), ("meteo", "start")], + ) + def test_a_date_that_does_not_match_its_format_is_refused( + self, distributed_mapping, block, field + ): + """Test that every date is checked against the format it is written in. + + Args: + distributed_mapping: A complete distributed configuration. + block: The block carrying the date. + field: The date field to corrupt. + + Test scenario: + The module promises a validated config is consumable without re-checking, but an + unparseable date slipped through to fail later inside `Catchment.__init__` -- or, + for the meteo bounds, deep in the loader. + """ + distributed_mapping[block][field] = "not-a-date" + + with pytest.raises(ValidationError, match="does not match its format") as exc: + RunConfig.model_validate(distributed_mapping) + + assert f"{block}.{field}" in str(exc.value), ( + f"the error should name the offending field: {exc.value}" + ) + + def test_a_period_that_ends_before_it_starts_is_refused(self, distributed_mapping): + """Test that a reversed period is caught at parse time. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + Nothing checked the order, so a reversed period produced an empty date index and + failed far downstream on a shape mismatch that said nothing about the dates. + """ + distributed_mapping["catchment"]["start"] = "2009-01-10" + distributed_mapping["catchment"]["end"] = "2009-01-01" + + with pytest.raises(ValidationError, match="is after"): + RunConfig.model_validate(distributed_mapping) + + def test_a_lumped_configuration_may_not_carry_a_flow_network(self, lumped_mapping): + """Test that a grid block on a lumped run is refused rather than ignored. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + `extra="forbid"` exists so a misspelled key fails loudly rather than being + dropped. Silently discarding a correctly spelled but inapplicable block is the + same silence by another route -- the user's block simply never took effect. + """ + lumped_mapping["flow_network"] = {"flow_accumulation": "acc.tif"} + + with pytest.raises(ValidationError, match="has no grid"): + RunConfig.model_validate(lumped_mapping) + + def test_a_lumped_configuration_may_not_name_a_grid_meteo_source(self, lumped_mapping): + """Test that a grid loader on a lumped run is refused. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + Lumped mode reads `meteo.path` with `pd.read_csv` regardless of `source`, so a + `source: netcdf` config would hand a `.nc` file to the CSV reader. + """ + lumped_mapping["meteo"]["source"] = "netcdf" + + with pytest.raises(ValidationError, match="does not apply"): + RunConfig.model_validate(lumped_mapping) + def test_parameters_and_gauges_may_both_be_omitted(self, distributed_mapping): """Test that a configuration carrying neither optional block still validates. @@ -789,6 +864,30 @@ def test_an_unregistered_model_class_is_refused_by_name( f"the error should list the known models: {exc.value}" ) + def test_an_unregistered_model_class_is_refused_before_the_readers_run( + self, distributed_mapping, tmp_path + ): + """Test that the registry lookup happens before any input is read. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The lookup needs nothing but the config, so running it after the parameter folder + read charged a typo the full cost of that I/O and left a partly-populated model + behind. Pointing every input at a path that does not exist isolates the ordering: + if the readers ran first the failure would name a missing file instead. + """ + distributed_mapping["conceptual_model"]["model_class"] = "HBV97" + distributed_mapping["parameters"]["path"] = "no/such/parameters" + distributed_mapping["meteo"]["path"] = "no/such/meteo.nc" + distributed_mapping["flow_network"]["flow_accumulation"] = "no/such/acc.tif" + distributed_mapping["flow_network"]["flow_direction"] = "no/such/fd.tif" + + with pytest.raises(ValueError, match="not.*registered"): + Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + @pytest.mark.parametrize("cls", [Catchment, Calibration]) def test_the_builder_returns_the_class_it_was_called_on( self, distributed_mapping, tmp_path, cls From d7c52641afe36805a12a61e9396d39a8ad7159f2 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:43:34 +0200 Subject: [PATCH 22/61] refactor(catchment): resolve the conceptual model before reading any input The registry lookup sat below `read_parameters`, so a typo'd `model_class` cost the whole parameter folder read before failing and left a partly populated model behind. It needs nothing but the config, so it now runs before the first reader -- the same reasoning the diff's own test states for validating the mapping up front. --- src/hapi/catchment.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index acc4c53c..36e43cb1 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -311,6 +311,16 @@ def from_yaml(cls, path: str) -> Self: routing_method=catchment.routing_method, ) + # Resolved before any reader runs: it needs nothing but the config, and a typo here + # would otherwise cost the whole parameter folder read before failing. + conceptual_model = config.conceptual_model + if conceptual_model.model_class not in CONCEPTUAL_MODELS: + raise ValueError( + f"conceptual_model.model_class {conceptual_model.model_class!r} is not " + f"registered; known models are {sorted(CONCEPTUAL_MODELS)}" + ) + model_class = CONCEPTUAL_MODELS[conceptual_model.model_class] + distributed = catchment.spatial_resolution == "distributed" if distributed: model.meteo = MeteoInputs.from_config( @@ -335,14 +345,8 @@ def from_yaml(cls, path: str) -> Self: maxbas=config.parameters.maxbas, ) - conceptual_model = config.conceptual_model - if conceptual_model.model_class not in CONCEPTUAL_MODELS: - raise ValueError( - f"conceptual_model.model_class {conceptual_model.model_class!r} is not " - f"registered; known models are {sorted(CONCEPTUAL_MODELS)}" - ) model.read_lumped_model( - CONCEPTUAL_MODELS[conceptual_model.model_class], + model_class, conceptual_model.catchment_area, conceptual_model.initial_condition, conceptual_model.q_init, From 859154eee4e9c7a8409e1d6cf9f742d4f6a8f239 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:45:05 +0200 Subject: [PATCH 23/61] fix(config): let a netcdf_files run pick its variable, and describe column truthfully `from_netcdf_files` takes a `variable` for files holding more than one, but `MeteoConfig` had no field for it and `from_config` never passed one, so such a user hit "pass variable= to pick one" with no way to comply from YAML. `GaugesConfig.column` claimed to select the discharge file names. It does not: `read_discharge_gauges` reads `.csv` regardless and only labels the resulting frame with `column`, so anything but "id" produces a frame whose declared columns are never written. The docstring now says what it does. `ConceptualModelConfig` also re-spelled `extra="forbid"` rather than extending `_STRICT`, which would have let a future change to the shared config skip that one class. --- src/hapi/config.py | 10 ++++++++-- src/hapi/inputs.py | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 7d6f0cfd..33f1eedb 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -129,6 +129,8 @@ class MeteoConfig(BaseModel): temperature: As `precipitation`, for temperature. evapotranspiration: As `precipitation`, for evapotranspiration. path: The combined NetCDF (`source="netcdf"`) or the lumped meteo CSV. + variable: Which variable to take from each file, `source="netcdf_files"` only. `None` + takes the single variable a file holds, which is an error if it holds several. start: Window start; `None` falls back to `catchment.start`. Distributed only. end: Window end; `None` falls back to `catchment.end`. Distributed only. fmt: `strptime` format for `start` / `end`. @@ -146,6 +148,7 @@ class MeteoConfig(BaseModel): temperature: str | None = None evapotranspiration: str | None = None path: str | None = None + variable: str | None = None start: str | None = None end: str | None = None fmt: str = "%Y-%m-%d" @@ -204,7 +207,7 @@ class ConceptualModelConfig(BaseModel): # `model_class` would collide with pydantic's protected `model_` namespace, so the namespace # is cleared rather than renaming a field the YAML already uses. - model_config = ConfigDict(extra="forbid", protected_namespaces=()) + model_config = ConfigDict(**_STRICT, protected_namespaces=()) model_class: str catchment_area: float = Field(gt=0) @@ -219,7 +222,10 @@ class GaugesConfig(BaseModel): discharge: Folder of one CSV per gauge id (distributed) or a single CSV (lumped). table: Gauge locations and properties. Distributed only; a lumped run has no grid to locate gauges on. - column: Gauge-table column holding the ids the discharge file names match. + column: Gauge-table column naming the resulting hydrograph columns. It does not + select the discharge file names: `read_discharge_gauges` reads `.csv` + regardless, so anything but `"id"` labels the frame with one set of names while + filling another. delimiter: Discharge CSV delimiter. fmt: `strptime` format for the discharge CSV's date column. """ diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index 20cd5087..85b12268 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -1298,6 +1298,7 @@ def from_config( precipitation, temperature, evapotranspiration, + variable=config.variable, start=window_start, end=window_end, fmt=config.fmt, From 3bff289b42ea758d3a11dec107c28feefefcc31b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:45:06 +0200 Subject: [PATCH 24/61] fix(catchment): accept a Path, and name the file when it is empty `from_yaml` annotated `path: str` while immediately wrapping it in `Path`, so a caller holding a `Path` had to stringify it -- and the rest of the package annotates such arguments `str | Path`. Its `Raises:` also listed only the validation errors, omitting the two most likely for the intended audience: a missing file and malformed YAML. An empty file was the worst of them, parsing to None and surfacing as pydantic's "Input should be a valid dictionary" without naming which file was empty. It now raises saying so. --- src/hapi/catchment.py | 18 +++++++++++++----- tests/test_config.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 36e43cb1..ffda3195 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -233,7 +233,7 @@ def __init__( self.config: RunConfig | None = None @classmethod - def from_yaml(cls, path: str) -> Self: + def from_yaml(cls, path: str | Path) -> Self: """Read a YAML run configuration and assemble a model from it. The alternate constructor for the build-then-mutate pattern this class documents: it @@ -251,16 +251,19 @@ def from_yaml(cls, path: str) -> Self: `Run.from_yaml` raises `TypeError` rather than silently building the wrong thing. Args: - path: Path to the YAML file. See :mod:`hapi.config` for the schema. + path: Path to the YAML file, as a string or a `Path`. See :mod:`hapi.config` for + the schema. Returns: Self: The model, with every input read, parsed and assigned. Raises: + FileNotFoundError: No file at `path`. + yaml.YAMLError: The file is not valid YAML. pydantic.ValidationError: The file is missing a required field, carries an unknown one, or breaks one of the cross-field rules in :class:`hapi.config.RunConfig`. - ValueError: `conceptual_model.model_class` names a model that is not in - `CONCEPTUAL_MODELS`. + ValueError: The file is empty, or `conceptual_model.model_class` names a model + that is not in `CONCEPTUAL_MODELS`. Examples: - Build a lumped model and inspect what the configuration gave it: @@ -298,7 +301,12 @@ def from_yaml(cls, path: str) -> Self: # non-ASCII catchment name or path mojibakes on a machine whose default is not UTF-8 # -- and does so silently, since the corrupted text is still valid YAML. text = Path(path).read_text(encoding="utf-8") - config = RunConfig.model_validate(yaml.safe_load(text)) + mapping = yaml.safe_load(text) + # An empty file parses to None, which pydantic would report as the opaque + # "Input should be a valid dictionary" without saying which file was empty. + if mapping is None: + raise ValueError(f"the run configuration at {path} is empty") + config = RunConfig.model_validate(mapping) catchment = config.catchment model = cls( diff --git a/tests/test_config.py b/tests/test_config.py index d93e301e..fc890a37 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1088,6 +1088,44 @@ def test_a_hand_built_model_has_no_configuration(self, coello_start_date, coello assert model.config is None, f"expected no configuration, got {model.config}" + def test_a_path_object_is_accepted(self, distributed_mapping, tmp_path): + """Test that the path may be a `Path`, not only a string. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The body immediately wraps the argument in `Path`, and the rest of the package + annotates such arguments `str | Path`, so a caller holding a `Path` should not + have to stringify it. + """ + write_yaml(distributed_mapping, tmp_path) + + model = Catchment.from_yaml(tmp_path / "config.yaml") + + assert model.name == "Coello", f"the config was not read from the Path: {model.name}" + + def test_an_empty_file_names_itself(self, tmp_path): + """Test that an empty configuration file is reported as such. + + Args: + tmp_path: pytest temporary directory. + + Test scenario: + An empty file parses to `None`, which pydantic reports as "Input should be a valid + dictionary" without naming the file -- unhelpful when a run names several. + """ + path = tmp_path / "empty.yaml" + path.write_text("", encoding="utf-8") + + with pytest.raises(ValueError, match="is empty") as exc: + Catchment.from_yaml(str(path)) + + assert "empty.yaml" in str(exc.value), ( + f"the error should name the file: {exc.value}" + ) + def test_an_invalid_configuration_fails_before_anything_is_read( self, distributed_mapping, tmp_path ): From 0d280e99413c509b8e74d1591d910fe89bc8bac2 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:46:41 +0200 Subject: [PATCH 25/61] test(inputs): cover the raster and netcdf-files branches of from_config Every test and both doctests used `source: netcdf`, leaving the other two dispatch branches and both defensive raises unexecuted. The gap mattered most for `rasters`: it is what the shipped MAXBAS example depends on, and it forwards seven keyword arguments plus two conditional ones, the exact shape of code where a mis-forwarded argument stays invisible until a user's file names stop parsing. Five tests over the bundled Coello fixtures: the raster branch against the three folders, the conditional pass-throughs, the per-driver NetCDF branch, and the two raises. `hapi.config` holds at 100% line and branch. --- tests/test_config.py | 135 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index fc890a37..3d9c1505 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -32,6 +32,7 @@ ParametersConfig, RunConfig, ) +from hapi.inputs import MeteoInputs from hapi.run import Run COMBINED_NC = "tests/rrm/data/coello/meteo.nc" @@ -637,6 +638,140 @@ def test_parameters_and_gauges_may_both_be_omitted(self, distributed_mapping): assert config.gauges is None, "gauges should be absent, not defaulted" +class TestMeteoInputsFromConfig: + """Tests for the `meteo.source` dispatch in `MeteoInputs.from_config`.""" + + def test_the_raster_source_reads_the_three_folders( + self, + coello_prec_path: str, + coello_temp_path: str, + coello_evap_path: str, + coello_start_date: str, + coello_end_date: str, + ): + """Test that `source: rasters` reads the folders and dates them. + + Args: + coello_prec_path: Rainfall raster folder. + coello_temp_path: Temperature raster folder. + coello_evap_path: Evapotranspiration raster folder. + coello_start_date: Simulation start date. + coello_end_date: Simulation end date. + + Test scenario: + This is the branch the shipped MAXBAS example depends on, and the one with the + most argument plumbing -- seven forwarded keywords plus two conditional ones -- + so a mis-forwarded argument would be invisible until a user's file names stopped + parsing. + """ + meteo = MeteoInputs.from_config( + MeteoConfig( + source="rasters", + precipitation=coello_prec_path, + temperature=coello_temp_path, + evapotranspiration=coello_evap_path, + file_name_data_fmt="%Y.%m.%d", + ), + start=coello_start_date, + end=coello_end_date, + ) + + assert meteo.shape == (13, 14, 10), f"unexpected grid or step count: {meteo.shape}" + assert meteo.time is not None, "the calendar should come from the file names" + assert meteo.time[0].strftime("%Y-%m-%d") == coello_start_date + + def test_the_raster_source_forwards_the_reader_arguments( + self, + coello_prec_path: str, + coello_temp_path: str, + coello_evap_path: str, + ): + """Test that `glob`, `per_variable` and `gdal_env` reach the loader. + + Args: + coello_prec_path: Rainfall raster folder. + coello_temp_path: Temperature raster folder. + coello_evap_path: Evapotranspiration raster folder. + + Test scenario: + The conditional pass-throughs are the easiest to drop silently. A `glob` that + matches nothing must surface as a read error, which proves it was forwarded + rather than ignored. + """ + config = MeteoConfig( + source="rasters", + precipitation=coello_prec_path, + temperature=coello_temp_path, + evapotranspiration=coello_evap_path, + file_name_data_fmt="%Y.%m.%d", + glob="*.nothing", + gdal_env={"GDAL_PAM_ENABLED": "NO"}, + per_variable={"temperature": {"glob": "*.tif"}}, + ) + + with pytest.raises((FileNotFoundError, ValueError)): + MeteoInputs.from_config(config) + + def test_the_netcdf_files_source_reads_one_file_per_driver( + self, coello_start_date: str, coello_end_date: str + ): + """Test that `source: netcdf_files` reads a separate file for each driver. + + Args: + coello_start_date: Simulation start date. + coello_end_date: Simulation end date. + + Test scenario: + The third branch, and the only one whose driver fields are per-driver *paths* + rather than folders or variable names -- so a branch that confused them would + still find files but read the wrong ones. + """ + meteo = MeteoInputs.from_config( + MeteoConfig( + source="netcdf_files", + precipitation="tests/rrm/data/coello/prec.nc", + temperature="tests/rrm/data/coello/temp.nc", + evapotranspiration="tests/rrm/data/coello/evap.nc", + ), + start=coello_start_date, + end=coello_end_date, + ) + + assert meteo.shape == (13, 14, 10), f"unexpected grid or step count: {meteo.shape}" + + def test_a_config_missing_a_driver_is_refused(self): + """Test that the defensive guard fires for a hand-built config. + + Test scenario: + `RunConfig` rejects such a configuration, so this only fires for a `MeteoConfig` + built directly -- which is exactly when the message naming the missing drivers is + the only thing the caller has to go on. + """ + with pytest.raises(ValueError, match="all three drivers") as exc: + MeteoInputs.from_config(MeteoConfig(source="rasters", precipitation="p")) + + assert "temperature" in str(exc.value), ( + f"the error should name what is unset: {exc.value}" + ) + + def test_the_netcdf_source_needs_a_path(self): + """Test that `source: netcdf` without a path is refused. + + Test scenario: + For this source the driver fields are variable names inside one file, so without + the file there is nothing to read them from. + """ + with pytest.raises(ValueError, match="must set meteo.path"): + MeteoInputs.from_config( + MeteoConfig( + source="netcdf", + precipitation="precipitation", + temperature="temperature", + evapotranspiration="evapotranspiration", + ) + ) + + class TestRoutingMethodNormalisation: """Tests for the `routing_method` canonicalisation in `Catchment.__init__`.""" From d4be1f8c6c8d679548199b77f9faeb60ebd7e28b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:48:23 +0200 Subject: [PATCH 26/61] docs(config): publish the schema reference and say where the examples run `hapi.config` is not an internal helper -- it is the user-facing file format, and its docstrings are the only description of the schema -- but it had no API page and no nav entry, so the reference was unreachable from the rendered site. Adds `docs/api/config.md` covering `RunConfig` and each block it nests, with a pointer to `Catchment.from_yaml`. The new doctests also read paths under `tests/` and `examples/`, which are not in the wheel. They stay executable, since that is what keeps them honest, but each Examples section now says the paths are repository fixtures rather than implying an installed user can run them as written. --- docs/api/config.md | 32 ++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + src/hapi/catchment.py | 4 ++++ src/hapi/inputs.py | 3 +++ 4 files changed, 40 insertions(+) create mode 100644 docs/api/config.md diff --git a/docs/api/config.md b/docs/api/config.md new file mode 100644 index 00000000..d020a4f1 --- /dev/null +++ b/docs/api/config.md @@ -0,0 +1,32 @@ +# Config + +The schema of a YAML run configuration. Each block below is one top-level key of the file; the +rules tying them together — which blocks a given `spatial_resolution` requires, and which it +refuses — live on `RunConfig`. + +Build a model from a file with +[`Catchment.from_yaml`](catchment.md#hapi.catchment.Catchment.from_yaml). + +## RunConfig +::: hapi.config.RunConfig + +## CatchmentConfig +::: hapi.config.CatchmentConfig + +## MeteoConfig +::: hapi.config.MeteoConfig + +## FlowNetworkConfig +::: hapi.config.FlowNetworkConfig + +## ParametersConfig +::: hapi.config.ParametersConfig + +## ConceptualModelConfig +::: hapi.config.ConceptualModelConfig + +## GaugesConfig +::: hapi.config.GaugesConfig + +## OutputsConfig +::: hapi.config.OutputsConfig diff --git a/mkdocs.yml b/mkdocs.yml index 9e10cca6..3f2bbf4a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -104,6 +104,7 @@ nav: - dev/Installation.md - API Reference: - api/catchment.md + - api/config.md - api/dem.md - api/calibration.md - api/inputs.md diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index ffda3195..7e0a8138 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -266,6 +266,10 @@ def from_yaml(cls, path: str | Path) -> Self: that is not in `CONCEPTUAL_MODELS`. Examples: + The configurations below ship with the Hapi repository, so these run from a + checkout rather than an installed wheel; point at your own file to try them + elsewhere. + - Build a lumped model and inspect what the configuration gave it: ```python >>> from hapi.catchment import Catchment diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index 85b12268..e42fea46 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -1184,6 +1184,9 @@ def from_config( only fires for a `MeteoConfig` built by hand. Examples: + The paths below are fixtures in the Hapi repository, so these run from a checkout + rather than an installed wheel; substitute your own file to try them elsewhere. + - Load a combined NetCDF by naming the variable each driver sits in: ```python >>> from hapi.config import MeteoConfig From cdb1f70fba3f64716df72ce2b2514b60053f1059 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:49:44 +0200 Subject: [PATCH 27/61] docs(catchment): document the datetime the date guards now accept `read_discharge_gauges`, `plot_hydrograph` and `save_results` gained `isinstance(x, str)` guards so a caller can pass the `datetime` their `str | dt.datetime` annotations already promised, but their docstrings still described the arguments as `str` only. Three public signatures widened without saying so; they now say so. --- src/hapi/catchment.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 7e0a8138..0e4e4d58 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -815,9 +815,12 @@ def read_discharge_gauges( Default is "%Y-%m-%d". split (bool, optional): True to subset the data between `start_date` and `end_date`. Default is False. - start_date (str, optional): Start date for subsetting. + start_date (str | dt.datetime, optional): Start date for + subsetting. A string is parsed with `fmt`; a datetime is + used as it is. Default is "". - end_date (str, optional): End date for subsetting. + end_date (str | dt.datetime, optional): End date for + subsetting. See `start_date`. Default is "". readfrom (str, optional): Number of rows to skip when reading the CSV. Default is "". @@ -1067,8 +1070,10 @@ def plot_hydrograph( r"""Plot simulated and observed hydrographs for a given gauge. Args: - start_date (str): Starting date for the plot. - end_date (str): End date for the plot. + start_date (str | dt.datetime): Starting date for the plot. A + string is parsed with `fmt`; a datetime is used as it is. + end_date (str | dt.datetime): End date for the plot. See + `start_date`. gauge (int): Index of the gauge in the GaugesTable. hapi_color (tuple | str, optional): Color of the simulated hydrograph. Default is "#004c99". @@ -1354,9 +1359,12 @@ def save_results( 5 - Soil moisture, 6 - Upper zone, 7 - Lower zone, 8 - Water content. For lumped mode, 5 saves all variables. Default is 1. - start (str, optional): Start date for the output period. + start (str | dt.datetime, optional): Start date for the + output period. A string is parsed with `fmt`; a datetime + is used as it is. If empty, uses the first index. Default is "". - end (str, optional): End date for the output period. If + end (str | dt.datetime, optional): End date for the output + period. See `start`. If empty, uses the last index. Default is "". path (str, optional): Path to the output directory (distributed) or file (lumped). Default is "". From b36b36681a170f8ccec10d0319e3327392f3bcce Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:56:04 +0200 Subject: [PATCH 28/61] test: close the last uncovered branches this round introduced Two lines added by the round's fixes had no test: the guard rejecting a datetime bound when `read_rasters` is ordering by index rather than date, and the constructor's `spatial_resolution` check, whose routing-method sibling was already pinned. `hapi.config` holds at 100% line and branch, and `from_yaml` and the constructor now have no uncovered line between them. What remains uncovered in `catchment.py` and `inputs.py` is pre-existing surface this branch does not touch -- the plotting and animation methods, and the `Inputs` parameter helpers. --- .../rrm/catchment/test_read_raster_inputs.py | 27 +++++++++++- tests/test_config.py | 43 ++++++++++++++----- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/tests/rrm/catchment/test_read_raster_inputs.py b/tests/rrm/catchment/test_read_raster_inputs.py index 6cd947bd..914d1cfa 100644 --- a/tests/rrm/catchment/test_read_raster_inputs.py +++ b/tests/rrm/catchment/test_read_raster_inputs.py @@ -14,6 +14,7 @@ from __future__ import annotations +import datetime as dt import re from pathlib import Path @@ -22,7 +23,7 @@ from pyramids.dataset import Dataset from hapi.catchment import Catchment -from hapi.inputs import FlowNetwork, MeteoInputs +from hapi.inputs import FlowNetwork, MeteoInputs, read_rasters CELL_SIZE = 4000.0 """Cell size in metres used by every synthetic raster here (matches the Coello grid).""" @@ -591,6 +592,30 @@ def test_empty_directory_error_names_the_path(self, tmp_path): str(empty), str(empty), str(empty), file_name_data_fmt="%Y.%m.%d" ) + @pytest.mark.parametrize("bound", ["start", "end"]) + def test_a_datetime_bound_needs_the_date_ordering(self, coello_prec_path, bound): + """Test that a datetime bound is refused when the rasters are ordered by index. + + Args: + coello_prec_path: A folder of dated rasters. + bound: Which bound to pass as a datetime. + + Test scenario: + With `date=False` the rasters are ordered by the number in their name, so the + bounds are indices and a datetime has no meaning -- `int()` would fail on it + several frames down, in a message naming neither the argument nor the mode. + """ + with pytest.raises(TypeError, match="needs date=True") as exc: + read_rasters( + coello_prec_path, + date=False, + **{bound: dt.datetime(2009, 1, 1)}, + ) + + assert "indices" in str(exc.value), ( + f"the error should say what the bounds mean in this mode: {exc.value}" + ) + class TestReadFlowDir: """Tests for the flow-direction half of ``FlowNetwork.from_rasters``.""" diff --git a/tests/test_config.py b/tests/test_config.py index 3d9c1505..fae8665d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -399,7 +399,9 @@ def test_muskingum_requires_a_flow_direction_raster(self, distributed_mapping): """ del distributed_mapping["flow_network"]["flow_direction"] - with pytest.raises(ValidationError, match="flow_network.flow_direction is required"): + with pytest.raises( + ValidationError, match="flow_network.flow_direction is required" + ): RunConfig.model_validate(distributed_mapping) def test_maxbas_does_not_require_a_flow_direction_raster(self, distributed_mapping): @@ -603,7 +605,9 @@ def test_a_lumped_configuration_may_not_carry_a_flow_network(self, lumped_mappin with pytest.raises(ValidationError, match="has no grid"): RunConfig.model_validate(lumped_mapping) - def test_a_lumped_configuration_may_not_name_a_grid_meteo_source(self, lumped_mapping): + def test_a_lumped_configuration_may_not_name_a_grid_meteo_source( + self, lumped_mapping + ): """Test that a grid loader on a lumped run is refused. Args: @@ -676,7 +680,9 @@ def test_the_raster_source_reads_the_three_folders( end=coello_end_date, ) - assert meteo.shape == (13, 14, 10), f"unexpected grid or step count: {meteo.shape}" + assert meteo.shape == (13, 14, 10), ( + f"unexpected grid or step count: {meteo.shape}" + ) assert meteo.time is not None, "the calendar should come from the file names" assert meteo.time[0].strftime("%Y-%m-%d") == coello_start_date @@ -737,7 +743,9 @@ def test_the_netcdf_files_source_reads_one_file_per_driver( end=coello_end_date, ) - assert meteo.shape == (13, 14, 10), f"unexpected grid or step count: {meteo.shape}" + assert meteo.shape == (13, 14, 10), ( + f"unexpected grid or step count: {meteo.shape}" + ) def test_a_config_missing_a_driver_is_refused(self): """Test that the defensive guard fires for a hand-built config. @@ -799,9 +807,7 @@ def test_any_casing_is_stored_canonically(self, given, stored): lower-case "muskingum" stored verbatim therefore routed every cell down the MAXBAS branch and raised `TypeError: 'NoneType' object is not subscriptable`. """ - model = Catchment( - "coello", "2009-01-01", "2009-01-10", routing_method=given - ) + model = Catchment("coello", "2009-01-01", "2009-01-10", routing_method=given) assert model.routing_method == stored, ( f"{given!r} should be stored as {stored!r}, got {model.routing_method!r}" @@ -821,6 +827,17 @@ def test_kinematic_is_accepted_for_the_flood_model(self): assert model.routing_method == "Kinematic" + def test_an_unknown_spatial_resolution_is_refused(self): + """Test that the sibling enumerated argument is validated the same way. + + Test scenario: + `spatial_resolution` selects which half of the build runs, so an unrecognised + value has no branch to take and must fail at construction rather than silently + choosing the lumped one. + """ + with pytest.raises(ValueError, match="'lumped' and 'distributed'"): + Catchment("coello", "2009-01-01", "2009-01-10", spatial_resolution="semi") + def test_an_unknown_routing_method_is_refused(self): """Test that an unrecognised routing method fails at construction. @@ -1179,7 +1196,9 @@ def test_a_non_ascii_name_survives_the_read(self, distributed_mapping, tmp_path) model = Catchment.from_yaml(str(path)) - assert model.name == "Río Coello", f"the name was corrupted on read: {model.name!r}" + assert model.name == "Río Coello", ( + f"the name was corrupted on read: {model.name!r}" + ) def test_the_configuration_stays_reachable_on_the_model( self, distributed_mapping, tmp_path @@ -1208,7 +1227,9 @@ def test_the_configuration_stays_reachable_on_the_model( "the flow-accumulation path should stay reachable for save_results" ) - def test_a_hand_built_model_has_no_configuration(self, coello_start_date, coello_end_date): + def test_a_hand_built_model_has_no_configuration( + self, coello_start_date, coello_end_date + ): """Test that `config` is None on a model that was not built from a file. Args: @@ -1239,7 +1260,9 @@ def test_a_path_object_is_accepted(self, distributed_mapping, tmp_path): model = Catchment.from_yaml(tmp_path / "config.yaml") - assert model.name == "Coello", f"the config was not read from the Path: {model.name}" + assert model.name == "Coello", ( + f"the config was not read from the Path: {model.name}" + ) def test_an_empty_file_names_itself(self, tmp_path): """Test that an empty configuration file is reported as such. From dc4d12a1f40155b26e1e68e5a551610b1447dabc Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 21:57:28 +0200 Subject: [PATCH 29/61] docs(config): describe the constraints routing_method and flow_network now carry Two schema docstrings still described the looser rules they had before this round. `routing_method` said only that it is assigned onto the model and that the entry point is the caller's choice, when it now also requires `flow_network.flow_direction` for Muskingum and must agree with `parameters.maxbas`. `flow_network` said it is "absent for lumped" where a lumped configuration carrying one is now refused rather than ignored. --- src/hapi/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 33f1eedb..27f84f90 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -103,8 +103,11 @@ class CatchmentConfig(BaseModel): spatial_resolution: `"lumped"` or `"distributed"`. Selects the shape of `meteo` and `gauges`, and whether `flow_network` is required. temporal_resolution: `"daily"` or `"hourly"`. - routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`; which - `Run.*` entry point actually routes with it is the caller's choice. + routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`, and + constrains two other blocks: Muskingum routes along the network so it requires + `flow_network.flow_direction`, and the method must agree with + `parameters.maxbas`. Which `Run.*` entry point actually routes with it is still + the caller's choice. """ model_config = _STRICT @@ -262,7 +265,8 @@ class RunConfig(BaseModel): derives them from the bounds handed to `read_parameters_bound` rather than reading a fitted set. gauges: The observed discharge. Omit for a run that is not scored against gauges. - flow_network: The routing network. Required for distributed, absent for lumped. + flow_network: The routing network. Required for a distributed run and refused for a + lumped one, which has no grid to put it on. outputs: Where to write results. """ From 48d556987ad6f5519e85413df75027225da7b949 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:37:14 +0200 Subject: [PATCH 30/61] fix(config): resolve relative paths against the configuration file A relative path in a run configuration was handed to the reader as written, so it resolved against the process working directory. A configuration only worked when invoked from the one directory its author happened to use, and moving the file broke every path in it -- the paths describe where the inputs sit relative to the configuration, not relative to whoever runs it. `_resolve_config_paths` now rewrites every path field against the configuration's own directory before the model is built, so a configuration is self-contained and runs from anywhere. The four Coello examples state their inputs relative to themselves and load their YAML from beside the script, the lumped ones name an explicit `outputs.results_dir` instead of writing into the working directory, and the NetCDF example carries its own `meteo.nc` under the examples data tree rather than reaching into the test fixtures. --- .../coello-distributed-model-run-maxbas.py | 4 +- .../coello-distributed-model-run-maxbas.yaml | 15 +-- .../coello-distributed-model-run-netcdf.py | 4 +- .../coello-distributed-model-run-netcdf.yaml | 16 +-- .../run/coello-lumped-model-run-maxbas.py | 8 +- .../run/coello-lumped-model-run-maxbas.yaml | 11 +- .../coello/run/coello-lumped-model-run.py | 8 +- .../coello/run/coello-lumped-model-run.yaml | 11 +- .../data/distributed_model/meteo.nc | Bin 0 -> 34222 bytes src/hapi/catchment.py | 46 ++++++++ tests/test_config.py | 104 ++++++++++++++++-- 11 files changed, 179 insertions(+), 48 deletions(-) create mode 100644 examples/hydrological-model/data/distributed_model/meteo.nc diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py index 54e36969..fb7e7146 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.py @@ -15,9 +15,7 @@ from hapi.run import Run # %% Load the configuration and build the model -Coello = Catchment.from_yaml( - "examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml" -) +Coello = Catchment.from_yaml(__file__.removesuffix(".py") + ".yaml") # %% Run the model """ diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml index 3cfc45ad..6b311298 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-maxbas.yaml @@ -1,3 +1,4 @@ +# Paths are relative to this file, so the run works from any working directory. catchment: name: Coello start: "2009-01-01" @@ -8,17 +9,17 @@ catchment: meteo: source: rasters - precipitation: examples/hydrological-model/data/distributed_model/prec - temperature: examples/hydrological-model/data/distributed_model/temp - evapotranspiration: examples/hydrological-model/data/distributed_model/evap + precipitation: ../../data/distributed_model/prec + temperature: ../../data/distributed_model/temp + evapotranspiration: ../../data/distributed_model/evap file_name_data_fmt: "%Y.%m.%d" # MAXBAS sends every cell straight to the outlet, so no flow-direction raster is read. flow_network: - flow_accumulation: examples/hydrological-model/data/distributed_model/GIS/acc4000.tif + flow_accumulation: ../../data/distributed_model/GIS/acc4000.tif parameters: - path: examples/hydrological-model/data/distributed_model/parameters_initial_maxbas + path: ../../data/distributed_model/parameters_initial_maxbas snow: false maxbas: true @@ -28,7 +29,7 @@ conceptual_model: initial_condition: [0, 5, 5, 5, 0] gauges: - table: examples/hydrological-model/data/distributed_model/stations/gauges.csv - discharge: examples/hydrological-model/data/distributed_model/stations/ + table: ../../data/distributed_model/stations/gauges.csv + discharge: ../../data/distributed_model/stations/ column: id fmt: "%Y-%m-%d" diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py index fe488a7e..6bc88a39 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py @@ -21,9 +21,7 @@ from hapi.run import Run # %% Load the configuration and build the model -Coello = Catchment.from_yaml( - "examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml" -) +Coello = Catchment.from_yaml(__file__.removesuffix(".py") + ".yaml") # %% Check the drivers actually came from the file and cover the model print(f"meteo grid + steps : {Coello.meteo.shape}") diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml index 2e1bc88e..56179dda 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.yaml @@ -1,3 +1,4 @@ +# Paths are relative to this file, so the run works from any working directory. catchment: name: Coello start: "2009-01-01" @@ -6,19 +7,20 @@ catchment: temporal_resolution: daily routing_method: muskingum +# One file per rainfall product, each holding all three drivers with the calendar inside it. meteo: source: netcdf - path: tests/rrm/data/coello/meteo.nc + path: ../../data/distributed_model/meteo.nc precipitation: precipitation temperature: temperature evapotranspiration: evapotranspiration flow_network: - flow_accumulation: tests/rrm/data/coello/gis/acc4000.tif - flow_direction: tests/rrm/data/coello/gis/fd4000.tif + flow_accumulation: ../../data/distributed_model/GIS/acc4000.tif + flow_direction: ../../data/distributed_model/GIS/fd4000.tif parameters: - path: tests/rrm/data/coello/parameters/muskingum + path: ../../data/distributed_model/Parameter set-Avg snow: false maxbas: false @@ -28,11 +30,11 @@ conceptual_model: initial_condition: [0, 5, 5, 5, 0] gauges: - table: tests/rrm/data/coello/calibration/gauges.csv - discharge: tests/rrm/data/coello/calibration + table: ../../data/distributed_model/stations/gauges.csv + discharge: ../../data/distributed_model/stations column: id fmt: "%Y-%m-%d" # Written beside the other example outputs rather than into the repository root. outputs: - results_dir: examples/hydrological-model/data/distributed_model/results/ + results_dir: ../../data/distributed_model/results diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py index 639c467f..374d94ac 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.py @@ -20,9 +20,7 @@ from hapi.run import Run # %% Load the configuration and build the model -Coello = Catchment.from_yaml( - "examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml" -) +Coello = Catchment.from_yaml(__file__.removesuffix(".py") + ".yaml") # %% Routing # RoutingFn = Routing.triangular_routing_2 @@ -54,10 +52,10 @@ fig, ax = Coello.plot_hydrograph(Coello.start, Coello.end, gaugei, title="Lumped Model") # %% Save Results -SaveTo = "examples/hydrological-model/data/lumped_model/" +SaveTo = Coello.config.outputs.results_dir StartDate = "2009-01-01" EndDate = "2010-04-20" -path = f"{SaveTo}{Coello.name}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" +path = f"{SaveTo}/{Coello.name}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" Coello.save_results(result=5, start=StartDate, end=EndDate, path=path) print(f"results written to : {path}") diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml index 0b4bf6ea..5336691a 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run-maxbas.yaml @@ -1,3 +1,4 @@ +# Paths are relative to this file, so the run works from any working directory. catchment: name: Coello start: "2009-01-01" @@ -8,12 +9,12 @@ catchment: # Lumped mode reads one CSV of catchment-average drivers, not a grid: columns are # [date, precipitation, ET, temperature], optionally followed by the long-term average. meteo: - path: examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv + path: ../../data/lumped_model/meteo_data-MSWEP.csv # A single parameter file rather than a folder of rasters. `maxbas: true` says the set carries # the triangular-routing parameter; the routing function itself is chosen at run time. parameters: - path: examples/hydrological-model/data/lumped_model/coello-lumped-parameters2022-03-13-maxbas.txt + path: ../../data/lumped_model/coello-lumped-parameters2022-03-13-maxbas.txt snow: false maxbas: true @@ -24,5 +25,9 @@ conceptual_model: # One discharge file, and no gauge table: locating gauges on a grid is a distributed concern. gauges: - discharge: examples/hydrological-model/data/lumped_model/Qout_c.csv + discharge: ../../data/lumped_model/Qout_c.csv fmt: "%Y-%m-%d" + +# Written beside the other example outputs rather than into the working directory. +outputs: + results_dir: ../../data/lumped_model diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run.py b/examples/hydrological-model/coello/run/coello-lumped-model-run.py index 6d79776c..83aa3fbb 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run.py +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run.py @@ -20,9 +20,7 @@ from hapi.run import Run # %% Load the configuration and build the model -Coello = Catchment.from_yaml( - "examples/hydrological-model/coello/run/coello-lumped-model-run.yaml" -) +Coello = Catchment.from_yaml(__file__.removesuffix(".py") + ".yaml") # %% Routing # RoutingFn = Routing.triangular_routing_2 @@ -54,10 +52,10 @@ fig, ax = Coello.plot_hydrograph(Coello.start, Coello.end, gaugei, title="Lumped Model") # %% Save Results -SaveTo = "examples/hydrological-model/data/lumped_model/" +SaveTo = Coello.config.outputs.results_dir StartDate = "2009-01-01" EndDate = "2010-04-20" -path = f"{SaveTo}Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" +path = f"{SaveTo}/Results-Lumped-Model_{str(dt.datetime.now())[0:10]}.txt" Coello.save_results(result=5, start=StartDate, end=EndDate, path=path) print(f"results written to : {path}") diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml b/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml index a37ecb1c..6eb7a47a 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run.yaml @@ -1,3 +1,4 @@ +# Paths are relative to this file, so the run works from any working directory. catchment: name: Coello start: "2009-01-01" @@ -8,11 +9,11 @@ catchment: # Lumped mode reads one CSV of catchment-average drivers, not a grid: columns are # [date, precipitation, ET, temperature], optionally followed by the long-term average. meteo: - path: examples/hydrological-model/data/lumped_model/meteo_data-MSWEP.csv + path: ../../data/lumped_model/meteo_data-MSWEP.csv # A single parameter file rather than a folder of rasters. parameters: - path: examples/hydrological-model/data/lumped_model/Coello_Lumped2021-03-08_muskingum.txt + path: ../../data/lumped_model/Coello_Lumped2021-03-08_muskingum.txt snow: false maxbas: false @@ -23,5 +24,9 @@ conceptual_model: # One discharge file, and no gauge table: locating gauges on a grid is a distributed concern. gauges: - discharge: examples/hydrological-model/data/lumped_model/Qout_c.csv + discharge: ../../data/lumped_model/Qout_c.csv fmt: "%Y-%m-%d" + +# Written beside the other example outputs rather than into the working directory. +outputs: + results_dir: ../../data/lumped_model diff --git a/examples/hydrological-model/data/distributed_model/meteo.nc b/examples/hydrological-model/data/distributed_model/meteo.nc new file mode 100644 index 0000000000000000000000000000000000000000..386273baee6ab5727e04a8e81f5d5d0c0e11dd6b GIT binary patch literal 34222 zcmeI433L=yy2mR-!WI-2MHI9qqvI$cc4r}pER~xO0tyKx!R6&4Hc3NTNq6jYP}EVL zqc|w?+y-|=8An_Ze6CL&MI9YL$9;Lv5DZ{g1jqH9(c}2;cdP!LkFu-Mok!)&J1=$k z=ezfQ-|v36Zrxfrr8YxzldJm;eRV*; zEY+9}3XI(j?x&?u^?fw|HDpjz8Bo&R^&&C@RwP zeLi1~PbU@R%5q^&VNqdmak0O!NHvZ%12O{{GF`gtUX7+c+A`;Y^J1#+$#N((l9|<2 zr<7IK)McGfQKJ{->nG{6YG>#dMZ!VdUsRcuT`FqIE6S@XSWH%Sd1X!cjMC}ll8QtZ zH0W~V4Z&EbUawiy5(_rz{$klvFx(Psn!6yFl|4fSmsh(6|61*ND0V{D?4-gg3iEtP z#}*alB^~Sc7bM@Ye_UaH@;w)h%S}2s*PmCMe6U;|6kI-i`m~ufRnsQpXl7-P%PTDM z=M`q>78m8@78K^@<`s;~%bs3dSy4N6R^_zXx~!&PEGmzYf41bU!z#+kW>t%_v+8nl ze6o3F^)z_|>#{1M!C?4;P<>-owl7;$R!lE9duRyGmmb-e}?sHPfmp>$3J;YBS1fr&h^uwG}+x7GTd*)*)|s_Mz|D9YV%Se90NrHohUEkO3<0L-cp73Fm`WmPlf z6{<3@Ff?B-kVse$gd6mxU_+?2Ne|T5%Vko(NN;JDKDl0^!Fj=GFkCP9Ek~Y8qPkqv zWt9eF7sxfQ7Z$C5>dl6QKI@>D>!(5D=k?SUX;K7vC3>+ zUj7>+GK2;YnKw^Wa`e(bW1uO}lB1Ue!hwc>K31=64F?vs#>VSa;aDIVij3D!YYol~ z*2|E{f=JWcP(YSi=gPn3^{s)1NHj-RC*=a!agIJU5Q>Fz^iyP~p?VoJQ+C@VZEb7x zycU(aOgbVh`m`1TWa3$J1qB=Qnpi*{9sSpG-cHETPY;HJ7qtc#1ZW;gORLVT^JR}M zEXpp*mxdN?*C{Q}Xn)b{*_pX?%S+e#%iG>?O$#=+%-4p=`>4DD%1zh$rO8c|>3(N! z=U%31nI#mHt__fsA8c(I9XnmE9p8qM_m)Tc8_Se9WWwrLRHz+Q!(NU#}S)*S~JAt4AbCHUGx3k!>Jnz z8le5xeABzm3d%f&21AElJz-SJOR^edf9a<|#zIX&wJWqf4wN$@*92x{a;PcPu+{Qhy3cpA`lo3hu7215^Hp@9?1VPh^x?E-60KIg znoQ=d3UiKcu;#ebY>uy{0glIJKddG>${wsHnq#>&Uj6JJ>F{Tapjw)FO3Ts#bY#=a z^T&D?&KOD|nl`~S>hF@5?>-ExN7F_eZn{phA~HuMUOkkXbfMG@M;vvic`!z?Q`$-; zADzvuCkLa{f-xGLD4re++O3pN?e6Ajk?eHUfsfHiC$~R8e8j(Z&H01YUN)IdEkj)v z(+-eVRkfvESb^)JtgmUi<|VNGwPlTFO#Ez@*QBIRWOD&sGN^2)yp;vyi$IQUZT>uM z>epMpqdb}A<|%uid8W^)ESnjPGzX)xki1ONsh%Z8J<;=^yoXQB%?_*U>BRgTzc0t1 z-Pka%U}A2LKgVaDVKKE%HEo7Da`pH7)7O2dX}eec{kq`?Yud@V%iGB=eK{-VFlj%k zed-dj$A5O}MzRYp*fR2PX7=L|Z^tXG)!oF!KM6|0^;%Jd7S%5 zS&va~l$xe%BL-?C(Nv@B)Blp9Es6q?&eTbeU@po zUJ591%C4_wP-g=Vp;~%cV-8vQvL{~@Xrnhg`eOdS`(lu}fc7a}J4hQw-yOPMq^D1- zsntfxJoJuDv#q`d$Zrpr4Op6u#-rsQxltRh(?R2P+VxoGZ$|U|ns2>CHcKCEnak+^ z253Y+8c}uml=ABG%Chn~YDDAY0iY4l=aH6RtZQfu@*ONpkHy+R@|(*=S6@poQdrC~ z9ZTA(jx;Umqet&s_pKUVLM`zTUSd9|@4pcuGi-Mwd}G#^MKp(afx)Y2+M@TYx9SXa zMA8lzVD5@GME+lde3zgv!a?kx@0|v@C*~Z{QKAjjK7w2BTlUx9Kh0NJr_evwY5v2C z(`vd-^ENt3w`d#M91YfonnUu#qFkL=Yvh?WY-STb6FxeHTK~Dv zaQXzu&uF@!^ExfRz#r23@%IUrBGaTm)ELN|Xuj>p)eayQX=9;TH zFDD zixuvOXvbF=>wB}h^-#a)2Tu`*M?coJFCLG6aIZ!@`mu%^k4Hc9%ZAzzkAAFX?Fe*; zhCbUj1AQ~lHv@e$&^H5pGtf5!eKXKE1AQ~lHv@e$&^H7B4`$%UpDz1pe`#;^pb4Yu zYn4*z^XUPLCpqcb#Mx|spiig0#y76FmKCif!-h9QVU-5ce{mkU%g6R&m2_lA+j_0T&;o+aYI*g5h!akt<8w#ow*`R!Km zEEoLUaftKEp0(~qWBBZqN~z?ya@bAAKfbuec+|KWsqAeM`eKgH*8Q=D6Pd`@II&`I8?tAmYH-Ir2GihYfi`uFJNqHMlms`Md!U2gc5k&xxz8dsXED zi~M#=JkJGxcO2rpvh63_Xq1%Pu9QlSH%8rL9G7v2QZgLHL*QWDcrZMlSKQtC%Z*Pi zT@ly&8}Bl>5{;7>ued`Gzu$n6`|*+28>Dr!Z;NZ~J*$j;wEbaYttH~X*g5h!ad)nM zQsn`Q{C2B&mJ9ywIK+8nUwQo|qdI$)QYtwtS;!c0> zD&vaE-^W$^+Pwx>qHz-A6?bgOV+MrWhaS7sAf3E;MO=YTRvBEsD_Uzn#DTGM}v1qj(4$tQ!x8=kto& zbkwEB1983g?Gpyq;=imnAmYH- zIr2GiZ3P=u9Y@OQ@{&MQ0hr1i$mp|2~YlH;xWUNLt5=`Tvja1;-LgLUJ< z@O)lz>u!9>c;Wm%#}%&JXmBMOCox`eqsP2uK*(Ku=6Zv)Wa(>h`JdZla9z6RZ37|> zjGZH&6SsZa`zjAu$K&l~H{Kr0>u%gT z_k20gNlu(oJlYd+@VVKZa@41NeYoY}KF;xWUZ!#JI+ekZ<-SDwzLe%5DIV>KIQZObPdV!I+MQ=Bp%1q_+{ZcI z&dby%uTvQuS?)uAyG0Deb06AyTW^+YSKU7eN+mBCvdM5f-d=X&?XkS>#)V(G%5ahs z=M<0jL>zo>wx=BRdFk03l+cG;9`55DZ|7y|lh>&Xjx6^fzuh8+;<*p)ysbCOf0_7V z36x4+Fl3YAc)Y#r#@l0g-HnT0K3q7-iF1ladm;`#H``N=`h54)8A|BGEf4o`j<@qN z^~vj021l0rkl$_*L-E{)cHY*T_<&7Yx~CI38~=yYco|UU%b~4r(iLk`w0? zkM=|yd~UX<9QApBW~Lya54Swr$2s24%hV^YQyCmt?n8dNMGPM!@7EsRTlGhFH3%q` zoM6Z%!|`}~*^Rfy^12)M@c1W%lbkrGc(fK5^D_0x>r@6umiv(3ZV^NA+=q7F)|=(izIvbpN+mBC zvdM5f-d=X&?XkS>#(jJ9!V)JraZd4QPsG9JW_!v}pHH5WtAsw>@^ByLcsnmspS(_G zaAdg;`Rx`l6wiHV=WV@NE<5GFH@hqPGxXpxexj67BPH`ykC2~Tb=)y)ua%85obTr zwO@ZgkA7H)|C|>6LJ_aiFBA1!JLUpDqM!Y|miz&|f&2C6weWjRu-*N<7Jgt(4pwdc z`JG4c5r1g?AXua2@4cV=Q@>Irv*KsiP{)W-e`&z`|LL|OQuVzR=@40krczhspyM`R$f?o(ul&IK+8nCm!;dxcc<%N~z?y;?(Wp&EIZQ zN`|9&2pp^%4~FOSiVOI*i~FzG7T2hiTLo95aT4Pd_tm-%0U`HkYqtrK@6D}oJ@lsz zv5%hkR6JyfI52jOd`{fFcebcJV3FT$70+_P-yMfIuk6<5-Qv}cOr?@z!{puK-sepv z!%;j04%Uqa!}EE?E&FDtXnT5BTn)$X7F>zONsL$AuA1EfLhh5Qb_&v|+qyK^ze{j! zs@NqU;=tHB@;Pxc@^+~_V3FT$iRZcC?~X&9SN5LTPLcDbsZ?^TtmqUa&zeexqj(4$ ztQ!x8=ktntS=}ih{a_ z>&AoO`MlyD-Lyvp#_Wv?^8%Y_oWyv=4XNBCAml#j-93WTc+}pwuK9S6;5ui*9sv;t z#?Fz?i90g(xyl0;`R$f?o(ul&IK+8n^@lpehJW3mluC{_7IlhWOx&TA3`g-0I9N9x z4A18k_g(sDqP(s%u5%CDA-EEalNhhKO_e(Ygxoh2d?rW@vpeJ3{%xn=di}^90wNBK zog<$Uw_;bP$^#bp?Us0+3;ymn#Cc`sUeqokk8V~qSC zd|q)&GPj5|KeorUvc6q#B^oC&UUA<=HVX*3PrF*a&q*D+s7nX!Xct^(X0!{4I52jO zd`{ewjLj+!Smd``;(0FkyWdk3`g-0I9N9x4A18k z_vy$EF?4#D-j`$MN;FPlyyD(Erb9r;y>e2AAl=^DrR7(32(FKpbO?wzFm{f7PTX%6 zn|Z(@zugkgbHU#ohd8fn!|FEi!?4XtspPo(!ZxvGyQyS2iig0#y76FmKCifE{`!fS z+T0db^rJSxm1vyAc*X5JcC&zxd*HA(LE3pkm-g;y6I_$VZx#@7VC)?EoVcQEH>*5g zk>75K=egkTjzgSRw)lm1v3{AURB~MJ+agN-(XNyXNAVCiST`OF&*v3)-+5a^+pS&t z;H)izE73TK@rnz6&@Le49{1%IK^pyimkwLGMR0vLWQ%}^17qjN=fpibzg^`4i~M#= NJkJGxcO2rl{U3Kvah?DG literal 0 HcmV?d00001 diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 0e4e4d58..49a65e44 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -37,6 +37,7 @@ from hapi.config import RunConfig from hapi.inputs import ( + METEO_VARIABLES, FlowNetwork, MeteoInputs, _warn_if_no_sentinel, @@ -80,6 +81,48 @@ } +def _resolve_config_paths(config: RunConfig, base: Path) -> None: + """Rewrite every relative path in a configuration against the file it came from. + + A configuration names its inputs relative to itself, so it runs from any working directory + and can be moved with the data it points at. Resolving against the process's working + directory instead would make a file valid only from the one place it happened to be written + for. An absolute path is left alone. + + `meteo`'s three driver fields are paths only for the folder and per-file sources; under + `source="netcdf"` they name variables inside `meteo.path` and must not be touched. + + Args: + config: The parsed configuration, rewritten in place. + base: Directory of the configuration file. + """ + + def resolve(value: str | None) -> str | None: + if value is None: + return value + return value if Path(value).is_absolute() else str((base / value).resolve()) + + if config.meteo.source != "netcdf": + for field in METEO_VARIABLES: + setattr(config.meteo, field, resolve(getattr(config.meteo, field))) + config.meteo.path = resolve(config.meteo.path) + + if config.flow_network is not None: + # `flow_accumulation` and the two below are required fields, so `resolve` cannot + # return None for them; the cast keeps that visible rather than widening the model. + config.flow_network.flow_accumulation = str( + resolve(config.flow_network.flow_accumulation) + ) + config.flow_network.flow_direction = resolve(config.flow_network.flow_direction) + if config.parameters is not None: + config.parameters.path = str(resolve(config.parameters.path)) + if config.gauges is not None: + config.gauges.table = resolve(config.gauges.table) + config.gauges.discharge = str(resolve(config.gauges.discharge)) + if config.outputs is not None: + config.outputs.results_dir = resolve(config.outputs.results_dir) + + @contextmanager def _name_the_path(path) -> Iterator[None]: """Re-raise a pyramids `FileNotFoundError` with the offending path in the message. @@ -311,6 +354,9 @@ def from_yaml(cls, path: str | Path) -> Self: if mapping is None: raise ValueError(f"the run configuration at {path} is empty") config = RunConfig.model_validate(mapping) + # Relative paths belong to the file, not to whatever directory the process happens to + # be in, so a configuration runs from anywhere and travels with the data it names. + _resolve_config_paths(config, Path(path).resolve().parent) catchment = config.catchment model = cls( diff --git a/tests/test_config.py b/tests/test_config.py index fae8665d..1c7ee30c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,6 +15,8 @@ from __future__ import annotations import copy +import os +from pathlib import Path import pytest import yaml @@ -116,8 +118,23 @@ def lumped_mapping( } +PATH_FIELDS = ( + ("meteo", "path"), + ("parameters", "path"), + ("gauges", "table"), + ("gauges", "discharge"), + ("flow_network", "flow_accumulation"), + ("flow_network", "flow_direction"), +) + + def write_yaml(mapping: dict, tmp_path) -> str: - """Dump a mapping to a YAML file. + """Dump a mapping to a YAML file, with its input paths made absolute. + + The fixtures name their inputs relative to the repository root, but `from_yaml` resolves a + relative path against the configuration's own directory -- and these configurations are + written to a temporary one. Absolutising here keeps each test about the field it is + exercising; the resolution rule itself is covered by its own test. Args: mapping: The configuration to write. @@ -126,6 +143,12 @@ def write_yaml(mapping: dict, tmp_path) -> str: Returns: str: Path to the written file. """ + mapping = copy.deepcopy(mapping) + for block, field in PATH_FIELDS: + value = mapping.get(block, {}).get(field) + if isinstance(value, str) and not Path(value).is_absolute(): + mapping[block][field] = str(Path(value).resolve()) + path = tmp_path / "config.yaml" path.write_text(yaml.safe_dump(mapping), encoding="utf-8") return str(path) @@ -1189,12 +1212,8 @@ def test_a_non_ascii_name_survives_the_read(self, distributed_mapping, tmp_path) and plot titles, and the same risk applies to every path field. """ distributed_mapping["catchment"]["name"] = "Río Coello" - path = tmp_path / "config.yaml" - path.write_text( - yaml.safe_dump(distributed_mapping, allow_unicode=True), encoding="utf-8" - ) - model = Catchment.from_yaml(str(path)) + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) assert model.name == "Río Coello", ( f"the name was corrupted on read: {model.name!r}" @@ -1220,9 +1239,9 @@ def test_the_configuration_stays_reachable_on_the_model( model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) assert model.config is not None, "the configuration should be kept on the model" - assert model.config.outputs.results_dir == "somewhere/else/", ( - f"outputs did not survive: {model.config.outputs}" - ) + assert model.config.outputs.results_dir.replace("\\", "/").endswith( + "somewhere/else" + ), f"outputs did not survive: {model.config.outputs}" assert model.config.flow_network.flow_accumulation is not None, ( "the flow-accumulation path should stay reachable for save_results" ) @@ -1244,6 +1263,66 @@ def test_a_hand_built_model_has_no_configuration( assert model.config is None, f"expected no configuration, got {model.config}" + def test_relative_paths_resolve_against_the_configuration_file( + self, distributed_mapping, tmp_path, monkeypatch + ): + """Test that a configuration runs from a working directory that is not its own. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + monkeypatch: Used to move the process out of the repository root. + + Test scenario: + Resolving against the process's working directory would make a file valid only + from the one place it happened to be written for. Rewriting the paths relative to + the config and then running from elsewhere is the case that distinguishes the two. + """ + repo_root = Path.cwd() + config_dir = tmp_path / "configs" + config_dir.mkdir() + for block, field in ( + ("meteo", "path"), + ("parameters", "path"), + ("gauges", "table"), + ("gauges", "discharge"), + ): + distributed_mapping[block][field] = os.path.relpath( + repo_root / distributed_mapping[block][field], config_dir + ) + for field in ("flow_accumulation", "flow_direction"): + distributed_mapping["flow_network"][field] = os.path.relpath( + repo_root / distributed_mapping["flow_network"][field], config_dir + ) + path = config_dir / "config.yaml" + path.write_text(yaml.safe_dump(distributed_mapping), encoding="utf-8") + + monkeypatch.chdir(tmp_path) + model = Catchment.from_yaml(str(path)) + + assert model.meteo is not None, "the drivers should resolve from the config's own dir" + assert model.flow_network is not None, "the network should resolve too" + + def test_a_netcdf_variable_name_is_not_treated_as_a_path( + self, distributed_mapping, tmp_path + ): + """Test that the driver fields survive path resolution when they name variables. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Under `source: netcdf` the three driver fields are variable names inside + `meteo.path`, not paths. Resolving them would turn `precipitation` into an + absolute directory and the read would fail looking for a variable of that name. + """ + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.config.meteo.precipitation == "precipitation", ( + f"the variable name was rewritten as a path: {model.config.meteo.precipitation}" + ) + def test_a_path_object_is_accepted(self, distributed_mapping, tmp_path): """Test that the path may be a `Path`, not only a string. @@ -1318,8 +1397,9 @@ def test_the_configuration_is_read_from_the_given_path( first = write_yaml(distributed_mapping, tmp_path) renamed = copy.deepcopy(distributed_mapping) renamed["catchment"]["name"] = "Elsewhere" - second = tmp_path / "other.yaml" - second.write_text(yaml.safe_dump(renamed), encoding="utf-8") + other_dir = tmp_path / "other" + other_dir.mkdir() + second = write_yaml(renamed, other_dir) assert Catchment.from_yaml(first).name == "Coello" - assert Catchment.from_yaml(str(second)).name == "Elsewhere" + assert Catchment.from_yaml(second).name == "Elsewhere" From 9b52f69ea0550f277e3d61bf45a070f2a6c8a6d1 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:38:44 +0200 Subject: [PATCH 31/61] refactor(catchment)!: restrict routing_method to the three known methods `Catchment.__init__` (and `Calibration.__init__` through it) used to store `routing_method` verbatim and accept any string. `distrrm.SpatialRouting` compares against `"Muskingum"` exactly, so every unrecognised spelling -- including the lower-case `"muskingum"` a YAML author writes -- silently took the non-Muskingum branch. The constructor now matches case-insensitively against the three known methods and stores the canonical spelling, which makes that comparison trustworthy. The canonicalisation was landed earlier on this branch as a `fix`, which understates it: the accompanying rejection is a public-contract change on the two constructors of a published package. This commit carries the marker the change needed and documents the accepted set in the Catchment API page, where a reader looking for the valid values will find it rather than only in a constructor docstring. BREAKING CHANGE: `Catchment` and `Calibration` now raise `ValueError` when `routing_method` is not one of `muskingum`, `maxbas` or `kinematic` (case-insensitive), instead of storing the string as given. Scripts passing any other spelling -- a descriptive label, a typo, or an empty string -- must be updated to one of the three. The stored value is now always the canonical `Muskingum` / `MAXBAS` / `Kinematic`, so code reading back `model.routing_method` sees the canonical spelling rather than the one it passed in. --- docs/api/catchment.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/api/catchment.md b/docs/api/catchment.md index cbf6054d..9ac34321 100644 --- a/docs/api/catchment.md +++ b/docs/api/catchment.md @@ -1,4 +1,26 @@ # Catchment +## Routing methods + +`Catchment` and `Calibration` accept exactly three routing methods, matched case-insensitively +and stored in the one spelling the internals compare against: + +| Written as | Stored as | Routes | +|---|---|---| +| `muskingum` | `Muskingum` | Cell to cell along the flow-direction network. | +| `maxbas` | `MAXBAS` | Every cell straight to the outlet through a triangular function. | +| `kinematic` | `Kinematic` | The flood model's own path (`Run.RunFloodModel`). | + +Anything else raises a `ValueError` naming the three. Up to and including version 1.7.0 the +constructor stored whatever string it was handed, so a run configured as `"Max_bas"` — or as a +descriptive label such +as `"Muskingum-Cunge"` — was accepted and then silently routed with Muskingum, because +`distrrm.SpatialRouting` compares against `"Muskingum"` exactly. Rejecting the spelling is what +makes that comparison trustworthy; a script passing a spelling outside the table has to be updated +to one of the three. + +A YAML run configuration reaches only the first two: `kinematic` selects the flood model, which +[`hapi.config`](config.md) does not describe. + ## Catchment ::: hapi.catchment.Catchment From 6a8abe6ea350811ad2a2cbf111186c3026010576 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:40:38 +0200 Subject: [PATCH 32/61] fix(catchment): fill QGauges by the label the frame was built with `read_discharge_gauges` labelled the frame from the gauge table's `column` but filled it with `self.QGauges[int(name)]`, taking `name` from the `id` column. The two agree only when `column` is `"id"`. Anything else -- the `name` column a table carries so its hydrographs read as station names -- produced a frame with the requested columns left entirely NaN and a second set of id-named columns appended beside them. Nothing raised, and `extract_discharge` then computed metrics over the phantom columns too. The labels are now tracked alongside the ids and each file fills the column its own row named, so `column` does what its documentation says. `column="id"` is unchanged. The schema's `gauges.column` no longer documents the footgun, because there is none. --- src/hapi/catchment.py | 16 ++++++++++------ src/hapi/config.py | 8 ++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 49a65e44..1c483493 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -855,8 +855,9 @@ def read_discharge_gauges( (distributed) or file (lumped). delimiter (str, optional): Delimiter between the date and the discharge column. Default is ",". - column (str, optional): Name of the column in the gauge - table containing the file names. Default is "id". + column (str, optional): Gauge-table column naming the columns of the + resulting `QGauges` frame. It does not select the file names -- + those always come from the "id" column. Default is "id". fmt (str, optional): Date format in the discharge files. Default is "%Y-%m-%d". split (bool, optional): True to subset the data between @@ -892,9 +893,12 @@ def read_discharge_gauges( "read_discharge_gauges in distributed mode" ) - self.QGauges = pd.DataFrame( - index=ind, columns=self.GaugesTable[column].tolist() - ) + # The frame is labelled from `column` but every file is named after `id`, so the + # two are tracked separately: filling by `int(name)` instead of by the label the + # frame was built with left a `column != "id"` table with the requested columns + # all-NaN and a second set of id-named ones beside them, silently. + labels = self.GaugesTable[column].tolist() + self.QGauges = pd.DataFrame(index=ind, columns=labels) for i in range(len(self.GaugesTable)): name = self.GaugesTable.loc[i, "id"] @@ -914,7 +918,7 @@ def read_discharge_gauges( ) f.index = [dt.datetime.strptime(i, fmt) for i in f.index.tolist()] - self.QGauges[int(name)] = f.loc[self.start : self.end, f.columns[-1]] + self.QGauges[labels[i]] = f.loc[self.start : self.end, f.columns[-1]] else: if not os.path.exists(path): raise FileNotFoundError( diff --git a/src/hapi/config.py b/src/hapi/config.py index 27f84f90..f9b432bb 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -225,10 +225,10 @@ class GaugesConfig(BaseModel): discharge: Folder of one CSV per gauge id (distributed) or a single CSV (lumped). table: Gauge locations and properties. Distributed only; a lumped run has no grid to locate gauges on. - column: Gauge-table column naming the resulting hydrograph columns. It does not - select the discharge file names: `read_discharge_gauges` reads `.csv` - regardless, so anything but `"id"` labels the frame with one set of names while - filling another. + column: Gauge-table column naming the columns of the resulting hydrograph frame. It + does not select the discharge file names -- `read_discharge_gauges` reads + `.csv` regardless -- so a table can label its hydrographs with human-readable + names while the files stay named after the ids. delimiter: Discharge CSV delimiter. fmt: `strptime` format for the discharge CSV's date column. """ From 5976a8602554940702aa597bcdfbf2e1c4d04857 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:42:09 +0200 Subject: [PATCH 33/61] feat(config): accept a date YAML has already parsed `start: 2009-01-01` written without quotes is a `datetime.date` by the time pydantic sees it, and the date fields are strings because the readers downstream parse them with `fmt`. The result was that the spelling a YAML author writes first was rejected with "Input should be a valid string" -- a message naming neither the field's expected form nor the quoting rule that fixes it. Every shipped example quotes its dates, so the trap only appeared once someone wrote their own file. A `mode="before"` validator on the two blocks that carry dates now renders a parsed date back out in the block's own `fmt`. That loses nothing: a date YAML has parsed has no format ambiguity left to preserve, so writing it in `fmt` is exactly what the quoted spelling would have said. Quoted dates, a custom `fmt`, and a full timestamp all keep working. --- src/hapi/config.py | 68 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index f9b432bb..4ee1986c 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -81,7 +81,7 @@ from __future__ import annotations -from datetime import datetime +from datetime import date, datetime from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -91,14 +91,48 @@ _STRICT = ConfigDict(extra="forbid") +def _write_dates_in_the_block_format(values: Any, fields: tuple[str, ...]) -> Any: + """Render any date YAML already parsed back into a string in the block's own format. + + An unquoted `start: 2009-01-01` is a `datetime.date` by the time pydantic sees it, and the + date fields are strings because the readers downstream parse them with `fmt`. Rejecting the + unquoted spelling would be rejecting the one a YAML author writes first, over a difference + that carries no information: a parsed date has no format ambiguity left to preserve, so it + is simply written back out in `fmt`. + + Args: + values: The raw mapping pydantic is about to validate. Anything else is passed through + for pydantic to reject with its own message. + fields: Names of the date fields in this block. + + Returns: + Any: The mapping, with any parsed date in `fields` replaced by its `fmt` rendering. + """ + if not isinstance(values, dict): + return values + + fmt = values.get("fmt", "%Y-%m-%d") + if not isinstance(fmt, str): + return values + + rendered = dict(values) + for field in fields: + value = rendered.get(field) + # `datetime` subclasses `date`, so this covers `2009-01-01 06:00:00` too. + if isinstance(value, date): + rendered[field] = value.strftime(fmt) + return rendered + + class CatchmentConfig(BaseModel): """The `Catchment` constructor arguments. Attributes: name: Catchment name. - start: Start date, parsed with `fmt`. Kept a string: the constructor does the parsing, - and an unquoted YAML date would arrive here already a `date`. - end: End date, parsed with `fmt`. + start: Start date, parsed with `fmt`. Held as a string, because the constructor does + the parsing; an unquoted YAML date arrives here already a `date` and is written + back out in `fmt`, so both spellings work. + end: End date, parsed with `fmt`. See `start`. fmt: `strptime` format for `start` / `end`. spatial_resolution: `"lumped"` or `"distributed"`. Selects the shape of `meteo` and `gauges`, and whether `flow_network` is required. @@ -120,6 +154,19 @@ class CatchmentConfig(BaseModel): temporal_resolution: Literal["daily", "hourly"] = "daily" routing_method: Literal["muskingum", "maxbas"] = "muskingum" + @model_validator(mode="before") + @classmethod + def _accept_a_date_yaml_already_parsed(cls, values: Any) -> Any: + """Render an unquoted YAML date back into `fmt` before the string fields see it. + + Args: + values: The raw mapping. + + Returns: + Any: The mapping, with `start` and `end` as strings. + """ + return _write_dates_in_the_block_format(values, ("start", "end")) + class MeteoConfig(BaseModel): """The meteorological drivers: a distributed grid or a lumped CSV. @@ -161,6 +208,19 @@ class MeteoConfig(BaseModel): per_variable: dict[str, dict[str, Any]] | None = None gdal_env: dict[str, str] | None = None + @model_validator(mode="before") + @classmethod + def _accept_a_date_yaml_already_parsed(cls, values: Any) -> Any: + """Render an unquoted YAML date back into `fmt` before the string fields see it. + + Args: + values: The raw mapping. + + Returns: + Any: The mapping, with `start` and `end` as strings. + """ + return _write_dates_in_the_block_format(values, ("start", "end")) + class FlowNetworkConfig(BaseModel): """The routing network. Distributed runs only. From 0d4aae23bebdcc320747f286a4bb79f22effe4c4 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:44:38 +0200 Subject: [PATCH 34/61] feat(config): refuse fields the chosen run shape never reads The lumped branch rejected `flow_network` and a non-raster `meteo.source` on the grounds that `extra="forbid"` exists so a misspelled key fails rather than being dropped, and that a correctly spelled but inapplicable block is the same silence by another route. That reasoning was then not applied to the rest: a lumped run silently accepted `gauges.table`, `gauges.column`, the three grid drivers and `meteo.start` / `meteo.end`, and a distributed one accepted every raster-only knob under a NetCDF source and `meteo.path` under either of the other two. Two cases raised, eight did not, which made the rule impossible to learn and left the author no signal that a line they wrote did nothing. Each `meteo.source` now declares the fields its branch of `MeteoInputs.from_config` actually reads, and anything set outside that set is named in the error. `lumped` does the same for `meteo` and `gauges`. Only explicitly set fields count -- `model_fields_set`, not the value -- so a default an author never wrote is never held against them, and the four shipped example configurations are unaffected. --- src/hapi/config.py | 109 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 4ee1986c..292259fc 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -15,7 +15,15 @@ Which fields are required therefore depends on `catchment.spatial_resolution` and, for a distributed run, on `meteo.source`. Those cross-field rules are enforced here by model -validators, so a `RunConfig` that validates is one the builder can consume without re-checking. +validators, so the builder can consume a validated `RunConfig` without re-deriving them. Two +things are still left to it: resolving `conceptual_model.model_class` against its registry, and +every path, which is not touched until a reader opens it. + +The same dependency decides which fields are *refused*: a field the chosen shape never reads is +rejected rather than dropped. `extra="forbid"` already does that for a misspelled key, and a +correctly spelled but inapplicable one is the same mistake -- a line in the file with no effect +-- so it fails the same way. Only fields written explicitly count, so defaults are never held +against an author. Lake-aware runs (`hapi.catchment.Lake`) and the flood model (`Run.RunFloodModel`) are out of scope -- both need inputs this schema does not carry. @@ -91,6 +99,85 @@ _STRICT = ConfigDict(extra="forbid") +#: Which `MeteoConfig` fields each `source` actually reaches, mirroring the three branches of +#: `MeteoInputs.from_config`. A field outside its source's set is one the reader never looks at, +#: so setting it says something the run will not do. +_METEO_FIELDS_BY_SOURCE: dict[str, frozenset[str]] = { + "rasters": frozenset( + { + "source", + "precipitation", + "temperature", + "evapotranspiration", + "start", + "end", + "fmt", + "glob", + "regex_string", + "file_name_data_fmt", + "per_variable", + "gdal_env", + } + ), + "netcdf": frozenset( + { + "source", + "path", + "precipitation", + "temperature", + "evapotranspiration", + "start", + "end", + "fmt", + } + ), + "netcdf_files": frozenset( + { + "source", + "precipitation", + "temperature", + "evapotranspiration", + "variable", + "start", + "end", + "fmt", + } + ), +} + +#: A lumped run reads `meteo.path` as one CSV of catchment-average drivers -- no grid, no +#: window, no reader knobs -- and locates nothing, so the gauge table and the column that names +#: hydrographs from it have nothing to act on. +_LUMPED_METEO_FIELDS = frozenset({"source", "path"}) +_LUMPED_GAUGES_FIELDS = frozenset({"discharge", "delimiter", "fmt"}) + + +def _reject_fields_the_run_will_not_read( + block: BaseModel, applicable: frozenset[str], block_name: str, because: str +) -> None: + """Refuse fields of a block that the chosen run shape never reads. + + `extra="forbid"` catches a misspelled key; this catches a correctly spelled one that does + not apply. Both are the same failure from the author's side -- a line in the file that has + no effect -- and silently dropping the second is what makes a configuration hard to trust. + Only explicitly set fields count, so a default the author never wrote is not held against + them. + + Args: + block: The block to check. + applicable: Names the run shape actually reads. + block_name: The block's key in the file, for the message. + because: Why the rest do not apply, phrased to follow "which". + + Raises: + ValueError: One or more set fields fall outside `applicable`. + """ + inapplicable = sorted(block.model_fields_set - applicable) + if inapplicable: + named = ", ".join(f"{block_name}.{name}" for name in inapplicable) + raise ValueError(f"{because}, so {named} would be read by nothing") + + def _write_dates_in_the_block_format(values: Any, fields: tuple[str, ...]) -> Any: """Render any date YAML already parsed back into a string in the block's own format. @@ -402,6 +489,12 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: f"{self.parameters.maxbas}; the parameter set and the routing method " f"must agree, or the run reads the wrong parameter as the routing one" ) + _reject_fields_the_run_will_not_read( + self.meteo, + _METEO_FIELDS_BY_SOURCE[self.meteo.source], + "meteo", + f"meteo.source is {self.meteo.source!r}", + ) else: if self.meteo.path is None: raise ValueError( @@ -422,6 +515,20 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: f"CSV of catchment-average drivers; meteo.source " f"{self.meteo.source!r} does not apply" ) + lumped = "catchment.spatial_resolution is 'lumped'" + _reject_fields_the_run_will_not_read( + self.meteo, + _LUMPED_METEO_FIELDS, + "meteo", + f"{lumped}, which reads meteo.path as one CSV of catchment-average drivers", + ) + if self.gauges is not None: + _reject_fields_the_run_will_not_read( + self.gauges, + _LUMPED_GAUGES_FIELDS, + "gauges", + f"{lumped}, which reads one discharge file and locates no gauges", + ) return self @model_validator(mode="after") From 429d19477ef0e929c3c110a2c983d8c85828de6a Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:47:28 +0200 Subject: [PATCH 35/61] fix(config): derive routing_method from the parameter set when it is unstated The routing-method / parameter-set agreement check ran only on the distributed branch, so the shipped lumped MAXBAS configuration -- `parameters.maxbas: true`, no `routing_method` -- took the `muskingum` default and built a model whose public `routing_method` said `Muskingum` while the script routed it with `triangular_routing_1`. Scoping the check to distributed is defensible, since a lumped run picks its routing function at the call site; leaving a wrong value on the attribute is not, because it is what `distrrm.SpatialRouting` keys off and the next reader will trust it. The check now runs for both resolutions, and an unwritten `routing_method` is derived from `parameters.maxbas` rather than left at its default -- the two describe the same choice from opposite sides. It runs before the block checks so that a distributed MAXBAS run can omit both `routing_method` and `flow_direction`, which MAXBAS never reads. A configuration with no `parameters` block keeps the default, having nothing to derive from. --- src/hapi/config.py | 62 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index 292259fc..6ff3ca1f 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -225,10 +225,11 @@ class CatchmentConfig(BaseModel): `gauges`, and whether `flow_network` is required. temporal_resolution: `"daily"` or `"hourly"`. routing_method: `"muskingum"` or `"maxbas"`. Assigned onto `model.routing_method`, and - constrains two other blocks: Muskingum routes along the network so it requires - `flow_network.flow_direction`, and the method must agree with - `parameters.maxbas`. Which `Run.*` entry point actually routes with it is still - the caller's choice. + constrains one other block: Muskingum routes along the network, so a distributed + run needs `flow_network.flow_direction`. Left unwritten it is derived from + `parameters.maxbas`, which describes the same choice from the parameter set's + side; written, it must agree with it. Which `Run.*` entry point actually routes + with it is still the caller's choice. """ model_config = _STRICT @@ -427,6 +428,45 @@ class RunConfig(BaseModel): flow_network: FlowNetworkConfig | None = None outputs: OutputsConfig | None = None + @model_validator(mode="after") + def _check_the_routing_method_matches_the_parameter_set(self) -> RunConfig: + """Make `routing_method` agree with the parameter set, deriving it where it is unstated. + + The parameter-count check downstream cannot catch a disagreement: a MAXBAS set holds 11 + parameters and a Muskingum set 12, and `parameters.maxbas` is what selects which count + is expected, so a set that contradicts the routing method still counts correctly. The + run then completes, reading the Muskingum X as the MAXBAS value (or K and X out of a + MAXBAS set), and produces a hydrograph that is quietly wrong. + + A lumped run picks its routing function at the call site rather than from this + attribute, so `routing_method` is not load-bearing there -- but it is public, it is what + `distrrm.SpatialRouting` keys off, and leaving it saying `Muskingum` on a run using a + MAXBAS parameter set would mislead the next reader. An unstated one is therefore + derived from the parameter set rather than left at its default. + + Returns: + RunConfig: This config, with `catchment.routing_method` filled in where it was + unstated and the parameter set says which it is. + + Raises: + ValueError: `catchment.routing_method` and `parameters.maxbas` disagree. + """ + if self.parameters is None: + return self + + if "routing_method" not in self.catchment.model_fields_set: + self.catchment.routing_method = "maxbas" if self.parameters.maxbas else "muskingum" + return self + + if (self.catchment.routing_method == "maxbas") != self.parameters.maxbas: + raise ValueError( + f"catchment.routing_method is {self.catchment.routing_method!r} but " + f"parameters.maxbas is {self.parameters.maxbas}; the parameter set and the " + f"routing method must agree, or the run reads the wrong parameter as the " + f"routing one" + ) + return self + @model_validator(mode="after") def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: """Enforce the fields each spatial resolution requires. @@ -475,20 +515,6 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: ) if self.meteo.source == "netcdf" and self.meteo.path is None: raise ValueError("meteo.source is 'netcdf', which needs meteo.path") - # The parameter-count check cannot catch a mismatch here: a MAXBAS set holds 11 - # parameters and a Muskingum set 12, and `parameters.maxbas` is what selects which - # count is expected -- so a set that disagrees with the routing method still counts - # correctly. The run then completes, reading the Muskingum X as the MAXBAS value - # (or K and X out of a MAXBAS set), and produces a hydrograph that is quietly wrong. - if self.parameters is not None: - wants_maxbas = self.catchment.routing_method == "maxbas" - if wants_maxbas != self.parameters.maxbas: - raise ValueError( - f"catchment.routing_method is " - f"{self.catchment.routing_method!r} but parameters.maxbas is " - f"{self.parameters.maxbas}; the parameter set and the routing method " - f"must agree, or the run reads the wrong parameter as the routing one" - ) _reject_fields_the_run_will_not_read( self.meteo, _METEO_FIELDS_BY_SOURCE[self.meteo.source], From 73354f4fa2cac536d734a20f8262881d56c6ed2d Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:48:34 +0200 Subject: [PATCH 36/61] feat(config): separate the gauge table's date format from the discharge one `gauges.fmt` was passed to both `read_gauge_table`, where it parses the table's optional `start` / `end` columns, and `read_discharge_gauges`, where it parses the discharge CSV index. Two independent files were forced to share one date layout, so a gauge table written differently from the discharge files could not be expressed at all -- and the field documented only one of its two effects, so anyone changing it to fix the discharge parse silently changed how the table was parsed. `gauges.table_fmt` now carries the table's format and falls back to `gauges.fmt` when unset, which is right whenever one hand wrote both. Nothing changes for a configuration that does not set it. --- src/hapi/catchment.py | 8 +++++++- src/hapi/config.py | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 1c483493..a73361af 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -414,8 +414,14 @@ def from_yaml(cls, path: str | Path) -> Self: gauges = config.gauges if gauges is not None: if distributed: + # The table's validity-period columns and the discharge files' index are two + # different files' date layouts, so they get two fields -- with the table + # falling back to the discharge format, which is right whenever one hand wrote + # both. model.read_gauge_table( - gauges.table, config.flow_network.flow_accumulation, fmt=gauges.fmt + gauges.table, + config.flow_network.flow_accumulation, + fmt=gauges.table_fmt or gauges.fmt, ) model.read_discharge_gauges( gauges.discharge, diff --git a/src/hapi/config.py b/src/hapi/config.py index 6ff3ca1f..f82243e6 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -379,6 +379,10 @@ class GaugesConfig(BaseModel): names while the files stay named after the ids. delimiter: Discharge CSV delimiter. fmt: `strptime` format for the discharge CSV's date column. + table_fmt: `strptime` format for the gauge table's optional `start` / `end` columns, + which bound each gauge's validity period. A separate field because the table is a + separate file that a separate hand may have written; `None` falls back to `fmt`, + which is right whenever the two were written together. """ model_config = _STRICT @@ -388,6 +392,7 @@ class GaugesConfig(BaseModel): column: str = "id" delimiter: str = "," fmt: str = "%Y-%m-%d" + table_fmt: str | None = None class OutputsConfig(BaseModel): From 90c073bfed3d654d7123bef4e09274298adff94c Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:50:08 +0200 Subject: [PATCH 37/61] fix(catchment): name the argument when a mode argument is not a string `spatial_resolution`, `temporal_resolution` and `routing_method` are all lower-cased in the constructor, so a non-string reached `.lower()` and raised an `AttributeError` naming neither the argument nor the class. `Calibration` made that reachable through its own signature, which declared all three `str | None` while passing them straight down -- an annotation advertising an input that crashes. The three are now checked together before any of them is lower-cased, raising a `TypeError` that names the argument and what it got, and `Calibration`'s annotations drop the `| None` they could never honour. --- src/hapi/calibration.py | 6 +++--- src/hapi/catchment.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/hapi/calibration.py b/src/hapi/calibration.py index 3b53093a..7a6eef2e 100644 --- a/src/hapi/calibration.py +++ b/src/hapi/calibration.py @@ -70,9 +70,9 @@ def __init__( start: str, end: str, fmt: str = "%Y-%m-%d", - spatial_resolution: str | None = "Lumped", - temporal_resolution: str | None = "Daily", - routing_method: str | None = "Muskingum", + spatial_resolution: str = "Lumped", + temporal_resolution: str = "Daily", + routing_method: str = "Muskingum", ): """Initialize the Calibration object. diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index a73361af..e9c5598c 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -184,6 +184,8 @@ def __init__( canonicalised. Default is "Muskingum". Raises: + TypeError: If `spatial_resolution`, `temporal_resolution` or + `routing_method` is not a string. ValueError: If `spatial_resolution` is not "lumped" or "distributed". ValueError: If `temporal_resolution` is not "daily" or @@ -195,6 +197,19 @@ def __init__( self.start = dt.datetime.strptime(start_data, fmt) self.end = dt.datetime.strptime(end, fmt) + # All three of the mode arguments are lower-cased below, so a non-string reaches + # `.lower()` and raises an `AttributeError` naming neither the argument nor the class. + # Checked together, once, rather than three times over. + for argument, value in ( + ("spatial_resolution", spatial_resolution), + ("temporal_resolution", temporal_resolution), + ("routing_method", routing_method), + ): + if not isinstance(value, str): + raise TypeError( + f"{argument} must be a string, got {type(value).__name__}" + ) + if spatial_resolution.lower() not in ["lumped", "distributed"]: raise ValueError( "available spatial resolutions are 'lumped' and 'distributed'" From b58b386522875058ef99bf67d2998b538e5d0dd3 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:50:42 +0200 Subject: [PATCH 38/61] docs(config): link the schema's routing methods to the constructor's `ROUTING_METHODS` accepts three methods and `CatchmentConfig.routing_method` names two, with nothing at either site saying the other exists. The gap is deliberate -- `kinematic` selects the flood model, whose inputs the schema does not carry -- but a reader hitting "Input should be 'muskingum' or 'maxbas'" on a value the constructor takes has no way to learn that from the code. Each site now points at the other and states the rule: the schema says why the third is unreachable, and the registry says that adding a method to it needs a decision in the schema. --- src/hapi/catchment.py | 2 ++ src/hapi/config.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index e9c5598c..adc3d865 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -74,6 +74,8 @@ #: canonicalises rather than storing what it was handed. `"Kinematic"` belongs here because #: that comparison is also how the flood model selects its own path: a non-Muskingum method #: with a real `bankfull_depth` skips the cell, which `Run.RunFloodModel` relies on. +#: `hapi.config.CatchmentConfig.routing_method` exposes the first two to YAML and says why the +#: third is not; a method added here needs a decision there too. ROUTING_METHODS = { "muskingum": "Muskingum", "maxbas": "MAXBAS", diff --git a/src/hapi/config.py b/src/hapi/config.py index f82243e6..ced5b523 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -240,6 +240,11 @@ class CatchmentConfig(BaseModel): fmt: str = "%Y-%m-%d" spatial_resolution: Literal["lumped", "distributed"] = "lumped" temporal_resolution: Literal["daily", "hourly"] = "daily" + # Two of the three keys of `hapi.catchment.ROUTING_METHODS`, which is what the constructor + # accepts. `kinematic` is left out deliberately: it selects the flood model, whose inputs + # (`read_river_geometry`, `bankfull_depth`) this schema does not carry, so a configuration + # naming it would validate and then build a model that cannot run. Adding a method there + # means deciding here whether the schema can describe a run that uses it. routing_method: Literal["muskingum", "maxbas"] = "muskingum" @model_validator(mode="before") From 808e630260064437974db76cca23c8e5456ee6bb Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 22:59:10 +0200 Subject: [PATCH 39/61] fix(routing): reject a MAXBAS below one in calculate_weights The NaN-maxbas hardening reached `triangular_routing_2` and the three conceptual models but not `calculate_weights`, which is where `triangular_routing_1` -- the function the lumped MAXBAS example and `DistMaxbas1` actually route with -- resolves its weights. Below one whole step the triangle has nothing to spread over: 0.5 produced a single weight and an all-zero hydrograph with nothing raised, 0 an `IndexError` about axis 1, and NaN "cannot convert float NaN to integer". The guard uses the same negated `not maxbas >= 1` form as the others, so NaN still fails here naming the parameter. It immediately caught a real one. The lake tests built their model from the Muskingum parameter set and then routed it triangularly, and `DistMaxbas1` reads `parameters[..., -1]` -- MAXBAS in a MAXBAS set, the Muskingum X in that one. Every cell routed with X = 0.2 and returned zeros, so four tests asserted their shapes and flags against output carrying nothing. They now build from the MAXBAS set, whose in-domain values run 1.4 to 2.4. The one test that drives both paths on a single model re-reads the Muskingum set for its Muskingum leg, since the two index different bands; it is still the same model object, which is what its flag assertion is about. --- src/hapi/routing.py | 17 +++++ tests/rrm/catchment/test_wrapper_with_lake.py | 68 ++++++++++++++++--- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/hapi/routing.py b/src/hapi/routing.py index dd508747..572ee7c4 100644 --- a/src/hapi/routing.py +++ b/src/hapi/routing.py @@ -258,12 +258,25 @@ def calculate_weights(maxbas): length is ``floor(MAXBAS)`` for integer values, or ``floor(MAXBAS) + 1`` for non-integer values. + Raises: + ValueError: If ``maxbas`` is less than 1. + Examples: >>> from hapi.routing import Routing >>> weights = Routing.calculate_weights(5) >>> print(weights) [0.08 0.24 0.36 0.24 0.08] """ + # The same guard `triangular_routing_2` and the three conceptual models carry, at the + # one point `triangular_routing_1` -- which the lumped MAXBAS example routes with -- + # passes through. Below 1 the triangle has no whole step to spread over: 0.5 produced a + # single weight and an all-zero hydrograph with nothing raised, 0 an `IndexError` about + # axis 1, and NaN "cannot convert float NaN to integer". `not maxbas >= 1` rather than + # `maxbas < 1` because both are false for NaN, and a calibration can produce one in a + # masked cell. + if not maxbas >= 1: + raise ValueError(f"Maxbas value has to be at least 1, got {maxbas}") + yant = 0 total = 0 # Just to verify how far from the unit is the result @@ -352,6 +365,10 @@ def triangular_routing_1(Q, MAXBAS): numpy.ndarray: Routed output hydrograph with the same length as ``Q``. + Raises: + ValueError: If ``MAXBAS`` is less than 1, raised by + `calculate_weights`. + Examples: >>> import numpy as np >>> from hapi.routing import Routing diff --git a/tests/rrm/catchment/test_wrapper_with_lake.py b/tests/rrm/catchment/test_wrapper_with_lake.py index 79f7c054..ee03c05b 100644 --- a/tests/rrm/catchment/test_wrapper_with_lake.py +++ b/tests/rrm/catchment/test_wrapper_with_lake.py @@ -63,6 +63,7 @@ def _build_coello( parameters: str, area: int, initial_cond: list, + maxbas: bool = False, ) -> Catchment: """Assemble a distributed Coello catchment with no run behind it. @@ -77,6 +78,9 @@ def _build_coello( parameters: Folder of distributed parameter rasters. area: Catchment area in km2. initial_cond: Initial HBV state. + maxbas: Whether `parameters` carries the triangular-routing parameter. The two sets + differ in their last band -- MAXBAS against the Muskingum X -- and `DistMaxbas1` + routes with whatever sits there, so a triangular run needs the MAXBAS set. Returns: Catchment: Model with `meteo`, `flow_network`, parameters, a lumped model and a @@ -100,7 +104,7 @@ def _build_coello( file_name_data_fmt="%Y.%m.%d", ) coello.flow_network = FlowNetwork.from_rasters(acc, fd) - coello.read_parameters(parameters, False) + coello.read_parameters(parameters, False, maxbas=maxbas) coello.read_lumped_model(HBVLumped, area, initial_cond) coello.flow_path_length_arr = _flow_path_length_like(coello) return coello @@ -173,6 +177,45 @@ def coello_no_lake( ) +@pytest.fixture +def coello_with_lake_inputs_maxbas( + coello_start_date: str, + coello_end_date: str, + coello_prec_path: str, + coello_temp_path: str, + coello_evap_path: str, + coello_acc_path: str, + coello_fd_path: str, + coello_dist_parameters_maxbas: str, + coello_cat_area: int, + coello_initial_cond: list, +) -> Catchment: + """Provide the same catchment carrying a MAXBAS parameter set. + + `DistMaxbas1` routes each cell with `parameters[..., -1]`, which is MAXBAS in this set and + the Muskingum X in the other. Handed the Muskingum set, every cell routed with X = 0.2 -- + below the one whole step a triangle needs -- and `triangular_routing_1` returned an all-zero + hydrograph without raising, so the triangular tests asserted their shapes and flags against + output that carried nothing. In-domain MAXBAS here runs 1.4 to 2.4. + + Returns: + Catchment: Model ready for a triangular lake-aware run. + """ + return _build_coello( + coello_start_date, + coello_end_date, + coello_prec_path, + coello_temp_path, + coello_evap_path, + coello_acc_path, + coello_fd_path, + coello_dist_parameters_maxbas, + coello_cat_area, + coello_initial_cond, + maxbas=True, + ) + + def _flow_path_length_like(model: Catchment) -> np.ndarray: """Build a flow-path-length grid masked to the catchment's active cells. @@ -320,7 +363,8 @@ def test_the_routed_lake_series_covers_every_simulation_step( def test_a_muskingum_lake_run_clears_a_flag_a_triangular_run_set( self, - coello_with_lake_inputs: Catchment, + coello_with_lake_inputs_maxbas: Catchment, + coello_dist_parameters_muskingum: str, coello_start_date: str, coello_end_date: str, ): @@ -329,9 +373,12 @@ def test_a_muskingum_lake_run_clears_a_flag_a_triangular_run_set( Test scenario: The flag tells `extract_discharge` that a cell of `Qtot` is a contribution rather than a discharge. Setting it by hand would test the assignment against itself, so - drive both paths on the same model in the order that makes the flag matter. + drive both paths on the same model in the order that makes the flag matter. The + two read different parameter layouts -- Muskingum takes bands 10 and 11, the + triangular path the last one -- so each leg is given the set it indexes into. The + model, and therefore the flag under test, is the same object throughout. """ - model = coello_with_lake_inputs + model = coello_with_lake_inputs_maxbas lake = _make_lake(model, coello_start_date, coello_end_date, seed=13) Wrapper.FW1Withlake(model, lake) @@ -339,6 +386,7 @@ def test_a_muskingum_lake_run_clears_a_flag_a_triangular_run_set( "the triangular path must mark the model before this test means anything" ) + model.read_parameters(coello_dist_parameters_muskingum, False) Wrapper.RRMWithlake(model, lake) assert model._maxbas_routed is False, ( @@ -351,7 +399,7 @@ class TestFW1WithLake: def test_fills_the_distributed_output_fields( self, - coello_with_lake_inputs: Catchment, + coello_with_lake_inputs_maxbas: Catchment, coello_start_date: str, coello_end_date: str, ): @@ -363,7 +411,7 @@ def test_fills_the_distributed_output_fields( has been through `FW1Withlake` and nothing else -- with a shared instance an earlier Muskingum run would have filled them and deleting the fix left this green. """ - model = coello_with_lake_inputs + model = coello_with_lake_inputs_maxbas lake = _make_lake(model, coello_start_date, coello_end_date, seed=17) assert model.Qtot is None, "the fixture must arrive with no run behind it" @@ -380,7 +428,7 @@ def test_fills_the_distributed_output_fields( def test_the_outlet_series_carries_the_lake_and_drops_the_extra_slot( self, - coello_with_lake_inputs: Catchment, + coello_with_lake_inputs_maxbas: Catchment, coello_start_date: str, coello_end_date: str, ): @@ -392,7 +440,7 @@ def test_the_outlet_series_carries_the_lake_and_drops_the_extra_slot( entry, so the lake series has to be trimmed the same way -- it was not, and the two could not be added at all. """ - model = coello_with_lake_inputs + model = coello_with_lake_inputs_maxbas lake = _make_lake(model, coello_start_date, coello_end_date, seed=19) Wrapper.FW1Withlake(model, lake) @@ -417,7 +465,7 @@ def test_the_outlet_series_carries_the_lake_and_drops_the_extra_slot( def test_marks_the_model_as_maxbas_routed( self, - coello_with_lake_inputs: Catchment, + coello_with_lake_inputs_maxbas: Catchment, coello_start_date: str, coello_end_date: str, ): @@ -428,7 +476,7 @@ def test_marks_the_model_as_maxbas_routed( cell of `Qtot` under-reports. The flag is what makes `extract_discharge` refuse rather than return the wrong hydrograph. """ - model = coello_with_lake_inputs + model = coello_with_lake_inputs_maxbas lake = _make_lake(model, coello_start_date, coello_end_date, seed=23) assert model._maxbas_routed is False, "a fresh model must start unflagged" From 3fd0b43d42309badbc291eb85dd9a002de023010 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:01:11 +0200 Subject: [PATCH 40/61] fix(catchment): join the results directory instead of concatenating it `save_results` built its raster names with `path + prefix + date`, so a directory written the normal way -- without a trailing separator -- produced `some/dirResult_2009-01-01.tif` beside the directory rather than inside it. The directory was also never created, and `path` was dereferenced with no type check even though `outputs.results_dir` is optional in a run configuration, so a caller forwarding it straight through hit `TypeError: unsupported operand type(s)` from a string concatenation rather than anything naming the argument. The names are now joined, the directory is created when it is missing, and a non-string `path` raises a `TypeError` that says what the argument means in each mode -- a directory when distributed, the CSV itself when lumped, which the docstring now states too. The NetCDF example stops assuming the optional `outputs` block is there and falls back to the working directory. --- .../coello-distributed-model-run-netcdf.py | 7 +++-- src/hapi/catchment.py | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py index 6bc88a39..400e3a14 100644 --- a/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py +++ b/examples/hydrological-model/coello/run/coello-distributed-model-run-netcdf.py @@ -79,8 +79,11 @@ # %% Save the routed discharge to rasters, one per time step # Both paths come from the configuration rather than being restated here: `save_results` # re-reads the flow-accumulation raster for georeferencing (FlowNetwork keeps only the arrays, -# not the source path), and `outputs.results_dir` says where the rasters go. -save_to = Coello.config.outputs.results_dir +# not the source path), and `outputs.results_dir` says where the rasters go. The block is +# optional, so a configuration without one writes beside the script rather than failing on a +# missing attribute. +outputs = Coello.config.outputs +save_to = (outputs.results_dir if outputs is not None else None) or "" Coello.save_results( flow_acc_path=Coello.config.flow_network.flow_accumulation, result=1, diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index adc3d865..800dad83 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -1439,8 +1439,9 @@ def save_results( end (str | dt.datetime, optional): End date for the output period. See `start`. If empty, uses the last index. Default is "". - path (str, optional): Path to the output directory - (distributed) or file (lumped). Default is "". + path (str, optional): Output directory (distributed, created + if it does not exist) or the CSV file itself (lumped). + Default is "", the working directory. prefix (str, optional): Prefix for the output file names. Default is "". fmt (str, optional): Date format for parsing `start` and @@ -1449,8 +1450,17 @@ def save_results( Raises: Exception: If `flow_acc_path` is not provided in distributed mode. + TypeError: If `path` is not a string. `outputs.results_dir` + is optional in a run configuration, so a caller + forwarding it can hold None. ValueError: If `result` is not a valid option. """ + if not isinstance(path, str): + raise TypeError( + f"path must be a string naming a directory (distributed) or a file " + f"(lumped), got {type(path).__name__}" + ) + if start == "": start = self.date_index[0] elif isinstance(start, str): @@ -1475,12 +1485,16 @@ def save_results( if prefix == "": prefix = "Result_" - # create a list of names - path = path + prefix - names = [path + str(i)[:10] for i in self.date_index[start_i:end_i]] - # names = [i.replace("-", "_") for i in names] - # names = [i.replace(" ", "_") for i in names] - names = [i + ".tif" for i in names] + # `path` names a directory here, unlike the lumped branch below where it is the + # CSV itself. Joined rather than concatenated: the old `path + prefix` wrote + # `some/dirResult_2009-01-01.tif` for any directory given without a trailing + # separator, which is how a directory is normally written. + if path and not os.path.isdir(path): + os.makedirs(path, exist_ok=True) + names = [ + os.path.join(path, f"{prefix}{str(i)[:10]}.tif") + for i in self.date_index[start_i:end_i] + ] if result == 1: arr = self.Qtot[:, :, start_i:end_i] elif result == 2: From 809aae55bd4fc60905ee658db485cb1abbda5ef5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:01:45 +0200 Subject: [PATCH 41/61] chore(examples): ignore the output the Coello examples write The four YAML-driven examples write into the same data tree they read from, so their results land beside tracked inputs in directories `.gitignore` did not cover -- ten `Result_*.tif` under the distributed model's results folder and four `*Results-Lumped-Model_*.txt` under the lumped one, all showing as untracked in every `git status` after a run. The inputs stay tracked; only the per-run artefacts are ignored. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index dc5279c1..1aa03c35 100644 --- a/.gitignore +++ b/.gitignore @@ -200,3 +200,9 @@ tests/mo/* /tests/rrm/data/test_results/prepare_inputs/7_precip_2009.01.08.tif /tests/rrm/data/test_results/prepare_inputs/8_precip_2009.01.09.tif /tests/rrm/data/test_results/prepare_inputs/9_precip_2009.01.10.tif + +# Output of the four YAML-driven Coello examples. They write into the same data tree they +# read from, so their results sit beside tracked inputs; the inputs stay tracked and the +# per-run artefacts do not. +/examples/hydrological-model/data/distributed_model/results/Result_*.tif +/examples/hydrological-model/data/lumped_model/*Results-Lumped-Model_*.txt From 206c296a0c7b4598a5c043956b14692ac46fda1b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:03:36 +0200 Subject: [PATCH 42/61] feat(config): name every missing input path before the first reader runs `model_class` was resolved ahead of the readers because a typo there would otherwise cost the whole parameter folder read before failing. The same argument applies to every path in the file, and none of them were checked: a misspelled gauge table was reported only after the meteorological cube and the parameter rasters had been read, which on a real grid is minutes to learn about one line. `from_yaml` now checks the paths the chosen shape will open -- the same set the schema validated against -- and raises one `FileNotFoundError` listing all of them with the field that named each. A configuration with two typos reports both at once, in hundredths of a second. Under `source="netcdf"` the three driver fields name variables inside the file rather than paths, so only the file itself is checked. --- src/hapi/catchment.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 800dad83..a0771ccb 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -125,6 +125,65 @@ def resolve(value: str | None) -> str | None: config.outputs.results_dir = resolve(config.outputs.results_dir) +def _check_the_configured_paths_exist(config: RunConfig, distributed: bool) -> None: + """Check every input path the run will open, before the first reader runs. + + The readers fail one at a time and in the order the build happens to call them, so a typo + in the gauge table is only reported after the whole meteorological cube and the parameter + folder have been read -- minutes, on a real grid, to learn about a line the file could have + been checked for at once. Every missing path is named together instead, so one pass over + the message fixes the file. + + Only the paths the chosen shape will actually open are checked, which is the same set the + schema validated the configuration against. + + Args: + config: The parsed configuration, with its paths already resolved. + distributed: Whether this is a distributed run. + + Raises: + FileNotFoundError: One or more configured paths do not exist. + """ + candidates: list[tuple[str, str | None]] = [] + if config.parameters is not None: + candidates.append(("parameters.path", config.parameters.path)) + + if distributed: + # Under `source="netcdf"` the three driver fields name variables inside `meteo.path`, + # not paths, so only the file itself is checked. + if config.meteo.source == "netcdf": + candidates.append(("meteo.path", config.meteo.path)) + else: + candidates += [ + (f"meteo.{name}", getattr(config.meteo, name)) + for name in METEO_VARIABLES + ] + if config.flow_network is not None: + candidates += [ + ("flow_network.flow_accumulation", config.flow_network.flow_accumulation), + ("flow_network.flow_direction", config.flow_network.flow_direction), + ] + else: + candidates.append(("meteo.path", config.meteo.path)) + + if config.gauges is not None: + candidates += [ + ("gauges.discharge", config.gauges.discharge), + ("gauges.table", config.gauges.table), + ] + + missing = [ + f"{field} -> {value}" + for field, value in candidates + if value is not None and not Path(value).exists() + ] + if missing: + raise FileNotFoundError( + "the run configuration names inputs that do not exist:\n " + + "\n ".join(missing) + ) + + @contextmanager def _name_the_path(path) -> Iterator[None]: """Re-raise a pyramids `FileNotFoundError` with the offending path in the message. @@ -397,6 +456,7 @@ def from_yaml(cls, path: str | Path) -> Self: model_class = CONCEPTUAL_MODELS[conceptual_model.model_class] distributed = catchment.spatial_resolution == "distributed" + _check_the_configured_paths_exist(config, distributed) if distributed: model.meteo = MeteoInputs.from_config( config.meteo, From b78b07dcd812c48e1c00f601d485a4fd21e466c7 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:04:54 +0200 Subject: [PATCH 43/61] fix(config): validate the resolved meteorological window, not the two pairs The date check compared `catchment.start`/`end` and `meteo.start`/`end` as two independent pairs and skipped a pair when either half was unset. But `MeteoInputs.from_config` mixes them -- each bound comes from `meteo` when stated and falls back to `catchment` otherwise -- so a `meteo` block stating only a start after the catchment's end gave an inverted effective window with neither pair inverted, and validated. The eventual failure was informative, but it is precisely the case this validator exists to catch. The window the run will actually use is now resolved the same way `from_config` resolves it and checked once, and the message says where each bound came from. A `meteo` window that is a genuine sub-period of the catchment is unaffected. --- src/hapi/config.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index ced5b523..1badccf3 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -592,14 +592,34 @@ def _check_the_dates_parse_and_are_ordered(self) -> RunConfig: f"{label} {value!r} does not match its format {fmt!r}: {error}" ) from error - for first, second, fmt, block in ( - (self.catchment.start, self.catchment.end, self.catchment.fmt, "catchment"), - (self.meteo.start, self.meteo.end, self.meteo.fmt, "meteo"), + if datetime.strptime( + self.catchment.start, self.catchment.fmt + ) > datetime.strptime(self.catchment.end, self.catchment.fmt): + raise ValueError( + f"catchment.start {self.catchment.start!r} is after catchment.end " + f"{self.catchment.end!r}" + ) + + # The window the run actually uses, not the two literal pairs: `MeteoInputs.from_config` + # takes each bound from `meteo` when it is stated and falls back to `catchment` + # otherwise, so a `meteo` block stating only an end can invert the effective window + # while neither pair is inverted on its own. + window_start, start_fmt = ( + (self.meteo.start, self.meteo.fmt) + if self.meteo.start is not None + else (self.catchment.start, self.catchment.fmt) + ) + window_end, end_fmt = ( + (self.meteo.end, self.meteo.fmt) + if self.meteo.end is not None + else (self.catchment.end, self.catchment.fmt) + ) + if datetime.strptime(window_start, start_fmt) > datetime.strptime( + window_end, end_fmt ): - if first is None or second is None: - continue - if datetime.strptime(first, fmt) > datetime.strptime(second, fmt): - raise ValueError( - f"{block}.start {first!r} is after {block}.end {second!r}" - ) + raise ValueError( + f"the meteorological window runs from {window_start!r} to {window_end!r}, " + f"which ends before it starts; each bound is taken from meteo when stated " + f"and from catchment otherwise" + ) return self From 20e48c7c95c439c6b01c3883883fabdb5e201040 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:07:06 +0200 Subject: [PATCH 44/61] feat(run): refuse Run.from_yaml with a message that names the pattern `Run` subclasses `Catchment` to hold its entry points, not to be a catchment, so the inherited `from_yaml` was a discoverable public classmethod whose only behaviour was to fail on constructor arity -- "__init__() got an unexpected keyword argument 'fmt'", which says nothing about `Run` being an unbound-method holder or about what to call instead. The override raises the same `TypeError` with a message that does: build the model with `Catchment.from_yaml` and pass it to `Run.RunHapi(model)`. --- src/hapi/catchment.py | 4 ++-- src/hapi/run.py | 24 +++++++++++++++++++++++- tests/test_config.py | 9 +++++---- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index a0771ccb..a246251d 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -366,8 +366,8 @@ def from_yaml(cls, path: str | Path) -> Self: Builds `cls`, so `Calibration.from_yaml(...)` returns a `Calibration` -- it takes the same constructor arguments. `Run` does not: it overrides `__init__` to take none, and - its entry points are called unbound on a catchment (`Run.RunHapi(model)`), so - `Run.from_yaml` raises `TypeError` rather than silently building the wrong thing. + its entry points are called unbound on a catchment (`Run.RunHapi(model)`), so it + overrides this method to refuse the call and say so. Args: path: Path to the YAML file, as a string or a `Path`. See :mod:`hapi.config` for diff --git a/src/hapi/run.py b/src/hapi/run.py index eeff4940..43f14d62 100644 --- a/src/hapi/run.py +++ b/src/hapi/run.py @@ -9,7 +9,8 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any +from pathlib import Path +from typing import Any, NoReturn import numpy as np import pandas as pd @@ -86,6 +87,27 @@ def __init__(self): """Initialize the Run class.""" self.Qsim: np.ndarray | pd.DataFrame | None = None + @classmethod + def from_yaml(cls, path: str | Path) -> NoReturn: + """Refuse to build a `Run`, explaining the pattern instead. + + `Run` subclasses `Catchment` to hold its entry points, not to be a catchment: its + `__init__` takes no arguments, so the inherited `Catchment.from_yaml` could only fail + with a `TypeError` about constructor arity -- an error saying nothing about what to do + instead. The methods here are called on a model built elsewhere. + + Args: + path: Ignored; present so the signature matches the one it overrides. + + Raises: + TypeError: Always. + """ + raise TypeError( + "Run cannot be built from a configuration; it holds the entry points that run a " + "model built elsewhere. Build the model with Catchment.from_yaml(path) and pass " + "it in, e.g. Run.RunHapi(model)." + ) + def RunHapi(self): """Run the distributed hydrological model. diff --git a/tests/test_config.py b/tests/test_config.py index 1c7ee30c..9e12e652 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1096,11 +1096,12 @@ def test_run_cannot_be_built_because_it_takes_no_constructor_arguments( Test scenario: `Run` inherits the classmethod but overrides `__init__` to take only `self`, and - its entry points are called unbound on a catchment (`Run.RunHapi(model)`). Pins - that the mismatch surfaces as a `TypeError` at the constructor rather than as a - half-built model, so the docstring's warning stays true. + its entry points are called unbound on a catchment (`Run.RunHapi(model)`). The + override refuses the call with a message naming that pattern, rather than letting + constructor arity produce a `TypeError` about an unexpected keyword argument -- + an error that says nothing about what to do instead. """ - with pytest.raises(TypeError, match="unexpected keyword argument"): + with pytest.raises(TypeError, match="Catchment.from_yaml"): Run.from_yaml(write_yaml(distributed_mapping, tmp_path)) def test_a_lumped_configuration_reads_the_averaged_driver_csv( From 74a80be040451746fbf12f788432b4c7554bc18a Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:07:26 +0200 Subject: [PATCH 45/61] build: cap pydantic at the major boundary `pydantic >=2.0` was open-ended, so a pydantic 3 release would be resolved into every new environment. `hapi.config` is written against pydantic 2's API -- `ConfigDict` spreading, `protected_namespaces=()`, `model_fields_set`, `Field(min_length=...)` on a list -- none of which a major release is obliged to keep, and the schema is the one place a silent behaviour change would be hardest to notice. The resolved version is unchanged, so no lockfile update follows. --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 58556142..5a01bcf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,10 @@ dependencies = [ "cleopatra >=0.32.0", "matplotlib >=3.11.0", "pyyaml >=6.0", - "pydantic >=2.0" + # Capped at the major boundary: `hapi.config` is built on pydantic 2's API -- ConfigDict + # spreading, `protected_namespaces=()`, `model_fields_set`, `Field(min_length=...)` on a + # list -- none of which a pydantic 3 is obliged to keep. + "pydantic >=2.0,<3" ] [project.optional-dependencies] From 0c42e45abe58cc163f3ea4f0b6fb9c717a8ab241 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:09:19 +0200 Subject: [PATCH 46/61] test(config): move the config tests beside the fixtures they use `tests/test_config.py` sat at the tests root while depending on `coello_acc_path` and `coello_dist_parameters_muskingum` from `tests/rrm/conftest.py` and on `lumped_parameters_path`, `lumped_meteo_data_path` and `lumped_gauges_path` from `tests/rrm/catchment/conftest.py`. It worked only through the `from tests.rrm..conftest import *` re-export chain, which inverts the direction that structure is meant to run in: a file at the root reaching down into the deepest conftest rather than a file beside it using what is already in scope. Moved to `tests/rrm/catchment/test_config.py`, where every fixture it uses is local or inherited normally. No test body changes. --- tests/{ => rrm/catchment}/test_config.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ => rrm/catchment}/test_config.py (100%) diff --git a/tests/test_config.py b/tests/rrm/catchment/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/rrm/catchment/test_config.py From e1e777e34b4102fe99561cf83871bb4fddf9cfeb Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:09:33 +0200 Subject: [PATCH 47/61] build: record the pydantic cap in the lockfile `pixi` re-resolved after the `<3` bound was added. The only change is the recorded constraint; no package version moves. --- pixi.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixi.lock b/pixi.lock index 8c3be14b..265f693a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -3425,7 +3425,7 @@ packages: - cleopatra>=0.32.0 - matplotlib>=3.11.0 - pyyaml>=6.0 - - pydantic>=2.0 + - pydantic>=2.0,<3 - earthlens[ecmwf]>=0.12.0 ; extra == 'inputs' requires_python: '>=3.11,<4' - pypi: https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl From 7017c30da790b21f7dac41e6023c9910135761f7 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:10:51 +0200 Subject: [PATCH 48/61] docs(config): correct three claims the code no longer supports Three pieces of prose had drifted from what the code does: The module docstring said a configuration that validates can be consumed "without re-checking", which read as a guarantee where it is an aspiration -- the builder still resolves `model_class` against its registry and still has to find the files. It now names both, and says where they happen. The out-of-scope list named only `Lake` and `RunFloodModel`. Also unreachable from the schema: `read_flow_path_length` and therefore `DistMaxbas2`, `read_river_geometry`, and reading a driver folder by numeric file order -- which is only reachable through `per_variable`, where it now meets the catchment window the schema always passes down and `read_rasters` refuses the pair. The `cls(...)` call in `from_yaml` passes its first three arguments positionally because `Catchment.__init__` names the second `start_data` and `Calibration.__init__` names it `start`. Tidying them into keywords would break `Calibration.from_yaml`, which the docstring advertises, so the constraint is now written down beside it. --- src/hapi/catchment.py | 4 ++++ src/hapi/config.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index a246251d..d674306c 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -435,6 +435,10 @@ def from_yaml(cls, path: str | Path) -> Self: _resolve_config_paths(config, Path(path).resolve().parent) catchment = config.catchment + # The first three go positionally on purpose: `Catchment.__init__` calls its second + # parameter `start_data` and `Calibration.__init__` calls it `start`, so naming them + # would break `Calibration.from_yaml` -- which this method is documented to support -- + # while still working here. Renaming the parameter is the fix, and is breaking. model = cls( catchment.name, catchment.start, diff --git a/src/hapi/config.py b/src/hapi/config.py index 1badccf3..a3659e08 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -16,8 +16,9 @@ Which fields are required therefore depends on `catchment.spatial_resolution` and, for a distributed run, on `meteo.source`. Those cross-field rules are enforced here by model validators, so the builder can consume a validated `RunConfig` without re-deriving them. Two -things are still left to it: resolving `conceptual_model.model_class` against its registry, and -every path, which is not touched until a reader opens it. +things are still left to it, because neither can be settled from the file alone: resolving +`conceptual_model.model_class` against the registry of model classes, and checking that the +paths exist. `Catchment.from_yaml` does both before it opens anything. The same dependency decides which fields are *refused*: a field the chosen shape never reads is rejected rather than dropped. `extra="forbid"` already does that for a misspelled key, and a @@ -25,8 +26,16 @@ -- so it fails the same way. Only fields written explicitly count, so defaults are never held against an author. -Lake-aware runs (`hapi.catchment.Lake`) and the flood model (`Run.RunFloodModel`) are out of -scope -- both need inputs this schema does not carry. +Out of scope, each because the schema carries no field that reaches it: + +- Lake-aware runs (`hapi.catchment.Lake`) and the flood model (`Run.RunFloodModel`), which need + a lake record and a river geometry respectively -- so `read_river_geometry` is unreachable. +- `read_flow_path_length`, and with it `DistMaxbas2`, which scales each cell's MAXBAS by its + distance to the outlet. +- Reading a driver folder by numeric file order rather than by date. `MeteoConfig` has no + `date` field, so `date=False` can only be reached through `per_variable` -- where it now + meets the catchment window this schema always passes down, and `read_rasters` refuses the + combination. Examples: - Validate a lumped configuration and read back what it holds: From 512b56d61e892c9197a54bcfca2a53aa05b23c2f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:15:13 +0200 Subject: [PATCH 49/61] refactor(config): phrase the two duplicated checks once `MeteoInputs.from_config` re-checks two rules `RunConfig` already enforces, which is right -- a `MeteoConfig` built by hand never passed through the schema -- but each site wrote its own sentence, and they had already drifted: "a distributed run needs all three drivers; meteo is missing ..." against "MeteoInputs needs all three drivers; the configuration leaves ... unset", and two different phrasings of the NetCDF path rule. Both messages now live in `hapi.config`, which `inputs` already imports, so the same rule reads the same way wherever it fires. The tests that pinned the old wording follow. --- src/hapi/config.py | 41 ++++++++++++++++++++++++------ src/hapi/inputs.py | 21 ++++++++------- tests/rrm/catchment/test_config.py | 8 +++--- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/hapi/config.py b/src/hapi/config.py index a3659e08..d55ab284 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -98,6 +98,7 @@ from __future__ import annotations +from collections.abc import Sequence from datetime import date, datetime from typing import Any, Literal @@ -161,6 +162,35 @@ _LUMPED_GAUGES_FIELDS = frozenset({"discharge", "delimiter", "fmt"}) +#: The three fields every distributed `meteo.source` needs, in the order they are reported. +METEO_DRIVERS = ("precipitation", "temperature", "evapotranspiration") + +#: What `source="netcdf"` needs beyond the drivers, since it reads all three from one file. +NETCDF_PATH_MESSAGE = ( + "meteo.source is 'netcdf', which reads the three drivers out of one file, so meteo.path " + "must be set" +) + + +def missing_drivers_message(missing: Sequence[str]) -> str: + """Phrase the "not all three drivers are named" error. + + `RunConfig` raises this for a configuration and `MeteoInputs.from_config` raises it again + for a `MeteoConfig` built by hand, which the schema never saw. Two sites, one rule -- so + the wording lives here rather than being written out twice and drifting. + + Args: + missing: Names of the unset drivers. + + Returns: + str: The message. + """ + return ( + f"a distributed run needs all three meteorological drivers; " + f"{', '.join(missing)} {'is' if len(missing) == 1 else 'are'} unset" + ) + + def _reject_fields_the_run_will_not_read( block: BaseModel, applicable: frozenset[str], block_name: str, because: str ) -> None: @@ -523,17 +553,12 @@ def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: "to locate the gauges on the grid" ) missing = [ - name - for name in ("precipitation", "temperature", "evapotranspiration") - if getattr(self.meteo, name) is None + name for name in METEO_DRIVERS if getattr(self.meteo, name) is None ] if missing: - raise ValueError( - f"a distributed run needs all three drivers; meteo is missing " - f"{', '.join(missing)}" - ) + raise ValueError(missing_drivers_message(missing)) if self.meteo.source == "netcdf" and self.meteo.path is None: - raise ValueError("meteo.source is 'netcdf', which needs meteo.path") + raise ValueError(NETCDF_PATH_MESSAGE) _reject_fields_the_run_will_not_read( self.meteo, _METEO_FIELDS_BY_SOURCE[self.meteo.source], diff --git a/src/hapi/inputs.py b/src/hapi/inputs.py index e42fea46..31a69b37 100644 --- a/src/hapi/inputs.py +++ b/src/hapi/inputs.py @@ -39,7 +39,11 @@ from pyramids.feature import FeatureCollection from pyramids.netcdf import NetCDF -from hapi.config import MeteoConfig +from hapi.config import ( + NETCDF_PATH_MESSAGE, + MeteoConfig, + missing_drivers_message, +) from hapi.dem import DEM @@ -1248,8 +1252,9 @@ def from_config( # The three are optional on the model because a lumped configuration sets none of them, # while every distributed source needs all three. `RunConfig` enforces that, so reaching - # the raise means a `MeteoConfig` was built by hand. Bound to locals so the check both - # reports what is missing and narrows the type for the calls below. + # the raise means a `MeteoConfig` was built by hand -- and it raises the same sentence, + # phrased once in `hapi.config`, because it is the same rule. Bound to locals so the + # check both reports what is missing and narrows the type for the calls below. precipitation = config.precipitation temperature = config.temperature evapotranspiration = config.evapotranspiration @@ -1257,10 +1262,7 @@ def from_config( missing = [ name for name in METEO_VARIABLES if getattr(config, name) is None ] - raise ValueError( - f"MeteoInputs needs all three drivers; the configuration leaves " - f"{', '.join(missing)} unset" - ) + raise ValueError(missing_drivers_message(missing)) if config.source == "rasters": extra: dict[str, Any] = {} @@ -1283,10 +1285,7 @@ def from_config( if config.source == "netcdf": if config.path is None: - raise ValueError( - "source 'netcdf' reads the three drivers out of one file, so the " - "configuration must set meteo.path" - ) + raise ValueError(NETCDF_PATH_MESSAGE) return cls.from_netcdf( config.path, precipitation=precipitation, diff --git a/tests/rrm/catchment/test_config.py b/tests/rrm/catchment/test_config.py index 9e12e652..1e7f5988 100644 --- a/tests/rrm/catchment/test_config.py +++ b/tests/rrm/catchment/test_config.py @@ -531,7 +531,7 @@ def test_distributed_requires_all_three_drivers(self, distributed_mapping, drive """ del distributed_mapping["meteo"][driver] - with pytest.raises(ValidationError, match="missing") as exc: + with pytest.raises(ValidationError, match="all three meteorological drivers") as exc: RunConfig.model_validate(distributed_mapping) assert driver in str(exc.value), ( @@ -550,7 +550,7 @@ def test_netcdf_source_requires_a_path(self, distributed_mapping): """ del distributed_mapping["meteo"]["path"] - with pytest.raises(ValidationError, match="needs meteo.path"): + with pytest.raises(ValidationError, match="meteo.path must be set"): RunConfig.model_validate(distributed_mapping) def test_lumped_requires_the_meteo_csv(self, lumped_mapping): @@ -778,7 +778,7 @@ def test_a_config_missing_a_driver_is_refused(self): built directly -- which is exactly when the message naming the missing drivers is the only thing the caller has to go on. """ - with pytest.raises(ValueError, match="all three drivers") as exc: + with pytest.raises(ValueError, match="all three meteorological drivers") as exc: MeteoInputs.from_config(MeteoConfig(source="rasters", precipitation="p")) assert "temperature" in str(exc.value), ( @@ -792,7 +792,7 @@ def test_the_netcdf_source_needs_a_path(self): For this source the driver fields are variable names inside one file, so without the file there is nothing to read them from. """ - with pytest.raises(ValueError, match="must set meteo.path"): + with pytest.raises(ValueError, match="meteo.path must be set"): MeteoInputs.from_config( MeteoConfig( source="netcdf", From 259258a7a3434d6c725e6da7753183f77f23bd2d Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:34:18 +0200 Subject: [PATCH 50/61] ci: run the docstring examples that actually execute The new `from_yaml` and `from_config` examples carry non-skipped doctests that read real data and assert on concrete values -- shapes, step counts, a canonical routing method. Nothing ran them: the whole-`src` doctest hook is commented out because 19 examples in the HBV modules are stale, and no task or CI step covered the rest. They would have drifted the first time the fixtures or a reader default changed, invisibly. A `doctests` pixi task runs the four modules whose examples pass -- config, catchment, inputs, routing: 24 examples, all green -- with a matching pre-commit hook and a step in the lint workflow's `static` job, which is the one with the dev environment. The whole-`src` hook stays disabled; widening the task is how the remaining modules get added as their examples are repaired. --- .github/workflows/lint.yml | 19 ++++++++++++++----- .pre-commit-config.yaml | 12 ++++++++++++ pyproject.toml | 8 ++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 648247a1..5f27a875 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -62,16 +62,16 @@ jobs: # fix to apply. run: pre-commit run --all-files --show-diff-on-failure --color=always env: - SKIP: no-commit-to-branch,mypy,pytest-check,notebook-check,pixi-lock-check + SKIP: no-commit-to-branch,mypy,pytest-check,notebook-check,pixi-lock-check,doctest-executable-modules # Type check. mypy needs the full `dev` pixi env (typed deps and an importable # package) but is neither platform- nor Python-version-sensitive, so one runner # covers it. It runs nowhere else in CI today. # - # No doctest step yet: the `>>>` examples in the HBV modules are stale and the - # matching pre-commit hook is disabled for the same reason. Add - # `pixi run -e dev pytest --doctest-modules src -p no:cacheprovider` here once they - # are repaired, and re-enable the hook alongside it. + # The doctest step below covers only the modules whose examples run; the `>>>` examples + # in the HBV modules are stale, which is why the whole-`src` pre-commit hook is still + # disabled. Widen the `doctests` task in pyproject.toml as those are repaired, and + # re-enable that hook once it covers everything. static: runs-on: ubuntu-latest timeout-minutes: 20 @@ -89,3 +89,12 @@ jobs: - name: Type check with mypy run: pixi run -e dev mypy + + # These examples assert on concrete values read from real data, so running them is + # what keeps them from drifting into prose. Needs the repo root as the working + # directory, which is where the paths in them are written from. + - name: Run the executable docstring examples + run: pixi run -e dev doctests + env: + HAPI_DATA_DIR: src/hapi/parameters + MPLBACKEND: Agg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45984056..3a89d13c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -140,6 +140,18 @@ repos: pass_filenames: false always_run: true + # The narrow half of the hook below: the modules whose examples do run. They assert on + # concrete values read from real data, so leaving them unexecuted means they drift silently. + # Add a module to the `doctests` task in pyproject.toml once its examples pass. + - repo: local + hooks: + - id: doctest-executable-modules + name: "[py - doctest] executable examples" + entry: pixi run --frozen -e dev doctests + language: system + pass_filenames: false + always_run: true + # - repo: local # hooks: # # Disabled until the docstring examples are repaired: 19 doctests in the diff --git a/pyproject.toml b/pyproject.toml index 5a01bcf0..fe246ef5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,6 +254,14 @@ description = "Run the main test suite" cmd = "pytest -vvv --cov=src/hapi -sv -m 'not fig_share' --cov-report=xml --cov-report=term-missing" description = "Run all test suite" +[tool.pixi.tasks.doctests] +# Not `src` as a whole: 19 doctests in the hbv / hbv_bergestrom92 / hbv_lake modules fail and +# are why the pre-commit doctest hook is commented out. These four are green, and their +# examples assert on concrete values read from real data, so they are worth running rather +# than leaving as prose that drifts. Add a module here once its examples pass. +cmd = "pytest --doctest-modules -p no:cacheprovider --no-cov src/hapi/config.py src/hapi/catchment.py src/hapi/inputs.py src/hapi/routing.py" +description = "Run the doctests of the modules whose examples are executable" + [tool.pixi.tasks.notebooks] cmd = "pytest --nbval --nbval-lax --verbose -p no:cacheprovider examples" description = "Check notebooks (they still need migration off the pre-rename Hapi package)" From 31d0c23c564b4f494ffd5108cbcca6fc79f3c8fe Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:34:58 +0200 Subject: [PATCH 51/61] style(pixi): wrap the doctests command within the line limit The task's `cmd` names four module paths on one line, which ran to 138 characters against the repository's 120-character limit. TOML's array form takes the same argument vector across four lines. --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fe246ef5..95c754e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -259,7 +259,11 @@ description = "Run all test suite" # are why the pre-commit doctest hook is commented out. These four are green, and their # examples assert on concrete values read from real data, so they are worth running rather # than leaving as prose that drifts. Add a module here once its examples pass. -cmd = "pytest --doctest-modules -p no:cacheprovider --no-cov src/hapi/config.py src/hapi/catchment.py src/hapi/inputs.py src/hapi/routing.py" +cmd = [ + "pytest", "--doctest-modules", "-p", "no:cacheprovider", "--no-cov", + "src/hapi/config.py", "src/hapi/catchment.py", + "src/hapi/inputs.py", "src/hapi/routing.py", +] description = "Run the doctests of the modules whose examples are executable" [tool.pixi.tasks.notebooks] From cd737e725ffb815d40b042fd77cc3ae0967709ed Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:36:50 +0200 Subject: [PATCH 52/61] docs: document the YAML run configuration The branch converted all four example scripts to `Catchment.from_yaml` but left the two documentation pages that mirror them teaching the hand-wired flow, and the only mention of the feature anywhere in `docs/` was the generated API stub. A reader learning Hapi from the documentation would not have found it, and the docs page and the example script for the same Coello lumped run no longer agreed on how a model is assembled. A new Examples page walks through one configuration end to end: a complete lumped file, what changes for a distributed run, the three `meteo.source` loaders, why paths resolve against the file, what the schema checks before anything is opened, and what stays out of scope. The two existing run pages link to it from the top, so both routes are discoverable from either. --- docs/examples/distributed-model-run.md | 7 ++ docs/examples/lumped-model-run.md | 7 ++ docs/examples/run-configuration.md | 167 +++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 182 insertions(+) create mode 100644 docs/examples/run-configuration.md diff --git a/docs/examples/distributed-model-run.md b/docs/examples/distributed-model-run.md index 76dfa169..59f8ecc3 100644 --- a/docs/examples/distributed-model-run.md +++ b/docs/examples/distributed-model-run.md @@ -1,4 +1,11 @@ # Distributed Hydrological Model + +!!! tip "Or drive it from a YAML file" + + Everything this page assembles in Python can live in a run configuration instead -- + one file holding the paths, dates and settings, read by `Catchment.from_yaml`. See + [Run configuration](run-configuration.md). + After preparing all the meteorological, GIS inputs required for the model, and Extracting the parameters for the catchment ```python diff --git a/docs/examples/lumped-model-run.md b/docs/examples/lumped-model-run.md index 45466ba6..f3dcb24b 100644 --- a/docs/examples/lumped-model-run.md +++ b/docs/examples/lumped-model-run.md @@ -1,4 +1,11 @@ # Lumped Model Run + +!!! tip "Or drive it from a YAML file" + + Everything this page assembles in Python can live in a run configuration instead -- + one file holding the paths, dates and settings, read by `Catchment.from_yaml`. See + [Run configuration](run-configuration.md). + To run the HBV lumped model inside Hapi you need to prepare the meteorological inputs (rainfall, temperature and potential evapotranspiration), HBV parameters, and the HBV model (you can load Bergström, 1992 version of HBV from Hapi ) - First load the prepared lumped version of the HBV module inside Hapi, the triangular routing function and the wrapper function that runs the lumped model `RUN`. diff --git a/docs/examples/run-configuration.md b/docs/examples/run-configuration.md new file mode 100644 index 00000000..98180ecb --- /dev/null +++ b/docs/examples/run-configuration.md @@ -0,0 +1,167 @@ +# Run Configuration (YAML) + +The other example pages assemble a model in Python: construct a `Catchment`, then call +`read_lumped_inputs`, `read_parameters`, `read_lumped_model` and the rest in the right order. +That works, but it puts every path, date and area inside the script, so a script is only ever +about one catchment — and the call order is something you have to know. + +A run configuration moves all of it into a YAML file that sits beside the data it names. +`Catchment.from_yaml` reads the file, validates it, and makes the same `read_*` calls in the +same order: + +```python +from hapi.catchment import Catchment +from hapi.run import Run + +Coello = Catchment.from_yaml("coello-lumped-model-run.yaml") +Run.runLumped(Coello, Routing.triangular_routing_1) +``` + +The four shipped examples under `examples/hydrological-model/coello/run/` are each a pair — a +`.py` that runs the model and a `.yaml` beside it holding everything the run needs. + +## A complete lumped configuration + +```yaml +# Paths are relative to this file, so the run works from any working directory. +catchment: + name: Coello + start: "2009-01-01" + end: "2011-12-31" + spatial_resolution: lumped + temporal_resolution: daily + +# Lumped mode reads one CSV of catchment-average drivers, not a grid: columns are +# [date, precipitation, ET, temperature], optionally followed by the long-term average. +meteo: + path: ../../data/lumped_model/meteo_data-MSWEP.csv + +parameters: + path: ../../data/lumped_model/Coello_Lumped2021-03-08_muskingum.txt + snow: false + maxbas: false + +conceptual_model: + model_class: HBVBergestrom92 + catchment_area: 1530 + initial_condition: [0, 10, 10, 10, 0] + +# One discharge file, and no gauge table: locating gauges on a grid is a distributed concern. +gauges: + discharge: ../../data/lumped_model/Qout_c.csv + fmt: "%Y-%m-%d" + +outputs: + results_dir: ../../data/lumped_model +``` + +`catchment`, `meteo` and `conceptual_model` are required. `parameters`, `gauges` and `outputs` +are optional: omit `parameters` for a calibration, which derives them from the bounds given to +`read_parameters_bound`, and omit `gauges` for a run that is not scored against observations. + +## What changes for a distributed run + +`spatial_resolution: distributed` changes the shape of two blocks and requires a third. `meteo` +becomes a grid, described by its `source`: + +```yaml +catchment: + name: Coello + start: "2009-01-01" + end: "2009-04-10" + spatial_resolution: distributed + routing_method: maxbas + +meteo: + source: rasters + precipitation: ../../data/distributed_model/prec + temperature: ../../data/distributed_model/temp + evapotranspiration: ../../data/distributed_model/evap + file_name_data_fmt: "%Y.%m.%d" + +# MAXBAS sends every cell straight to the outlet, so no flow-direction raster is read. +flow_network: + flow_accumulation: ../../data/distributed_model/GIS/acc4000.tif + +gauges: + table: ../../data/distributed_model/stations/gauges.csv + discharge: ../../data/distributed_model/stations/ +``` + +`meteo.source` picks which `MeteoInputs` loader builds the grid, and what the three driver +fields mean: + +| `source` | The three driver fields name | Reads | +|---|---|---| +| `rasters` (default) | A folder of dated GeoTIFFs each | `MeteoInputs.from_rasters` | +| `netcdf_files` | One NetCDF each | `MeteoInputs.from_netcdf_files` | +| `netcdf` | A variable inside `meteo.path` | `MeteoInputs.from_netcdf` | + +The last is the fastest: one file, opened once, with the calendar inside it. See +[Meteorological inputs](meteo-inputs.md) for how to pack a folder of rasters into one. + +## Paths are relative to the file + +A relative path in a configuration is resolved against the configuration's own directory, not +against whatever directory you happen to run from. That is what makes a configuration portable: +it travels with the data it names, and the run works from anywhere. Absolute paths are used as +written. + +The example scripts rely on this — each loads the YAML sitting next to it: + +```python +Coello = Catchment.from_yaml(__file__.removesuffix(".py") + ".yaml") +``` + +## What the file is checked for + +The file is validated in full before anything is opened, so a mistake is reported as a mistake +in the file rather than as a failure deep inside a reader: + +- **Unknown keys are refused.** A misspelled `precipitaton` fails at parse time instead of being + dropped and reappearing as a missing input. +- **So are keys that do not apply.** A `flow_network` block on a lumped run, `glob` under + `source: netcdf`, `gauges.table` on a lumped run — each is a line that would do nothing, and + each is named in the error. Only keys you actually wrote count; defaults are never held + against you. +- **Required blocks are checked per shape.** A distributed run needs `flow_network` and all + three drivers; Muskingum additionally needs `flow_network.flow_direction`, which MAXBAS never + reads. A lumped run needs `meteo.path`. +- **`routing_method` must agree with `parameters.maxbas`.** The two parameter counts differ by + one and `maxbas` selects which is expected, so a disagreeing pair still counts correctly and + then reads the wrong parameter as the routing one. Leave `routing_method` out and it is + derived from the parameter set. +- **Every date is parsed against its own `fmt`**, and the period must run forwards — including + the meteorological window, whose bounds fall back to the catchment's when unstated. +- **Every path is checked for existence** before the first reader runs, and all the missing ones + are reported together. + +Dates may be quoted or not: `start: 2009-01-01` is a date to YAML, and it is written back out in +the block's `fmt`. + +## Reading the configuration back + +The parsed configuration stays on the model as `model.config`, so the blocks the build does not +itself consume remain reachable — `outputs` above all: + +```python +outputs = Coello.config.outputs +save_to = (outputs.results_dir if outputs is not None else None) or "" +Coello.save_results( + flow_acc_path=Coello.config.flow_network.flow_accumulation, + result=1, + path=save_to, +) +``` + +## Out of scope + +The schema describes a `Catchment` run. It carries no field for a lake record, a river geometry, +or a flow-path-length raster, so lake-aware runs (`Run.RunHapiwithLake`), the flood model +(`Run.RunFloodModel`) and `DistMaxbas2` are still assembled in Python. + +`Calibration.from_yaml` works — it takes the same constructor arguments — and gives back a +`Calibration` to call the calibration methods on. `Run.from_yaml` does not: `Run` holds entry +points called on a model built elsewhere, so it refuses and says so. + +The full field-by-field reference is on the [Config API page](../api/config.md). diff --git a/mkdocs.yml b/mkdocs.yml index 3f2bbf4a..4dd268d5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -126,6 +126,7 @@ nav: - examples/meteo-inputs.md - examples/muskingum.md - examples/parameters.md + - examples/run-configuration.md - Change logs: change-log.md extra: From b0e08d83cd5bc0b772c7dd3103ecc190867cf897 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:53:46 +0200 Subject: [PATCH 53/61] test(config): cover the round-2 surface and close the review's test gaps Every behaviour changed in round two now has a test, and `hapi.config` reaches 100% line and branch coverage (was 94%). Schema: the unquoted-YAML-date normaliser over a date, a timestamp and a custom format, plus the two inputs it defers to pydantic on; each inapplicable field a lumped run and each `meteo.source` now refuse, and the counterpart that a default the author never wrote is not held against them; `routing_method` derived from `parameters.maxbas` both ways, the agreement check now running for lumped too, and the ordering that lets a derived MAXBAS run omit the flow-direction raster; the resolved meteorological window in its three inverted shapes and one valid sub-period; `gauges.table_fmt` falling back. Builder: the three documented `from_yaml` raises that had no test -- a missing file, malformed YAML, and a top-level scalar; the pre-flight check naming two missing paths at once, and not checking a NetCDF variable name as a path; the raster source, which every other `from_yaml` test skips by driving from one combined NetCDF; `gauges.column` labelling the frame with no column left NaN; the three constructor mode-argument `TypeError`s, through `Calibration` as well. Elsewhere: `save_results` joining a directory written without a separator, creating a nested one, and refusing a non-string path; the `calculate_weights` lower bound over 0.5, 0 and NaN, reaching `triangular_routing_1`, with the boundary value still routing; and a direct success test for `MeteoInputs.from_config` under `source="netcdf"`, which was covered only transitively. --- tests/rrm/catchment/test_config.py | 633 ++++++++++++++++++ .../catchment/test_maxbas_routing_variants.py | 65 ++ .../test_save_results_distributed.py | 81 +++ 3 files changed, 779 insertions(+) diff --git a/tests/rrm/catchment/test_config.py b/tests/rrm/catchment/test_config.py index 1e7f5988..98e9f1ba 100644 --- a/tests/rrm/catchment/test_config.py +++ b/tests/rrm/catchment/test_config.py @@ -15,6 +15,7 @@ from __future__ import annotations import copy +import datetime as dt import os from pathlib import Path @@ -227,6 +228,72 @@ def test_dates_stay_strings(self): ) assert config.start == "2009-01-01", f"start was altered: {config.start}" + @pytest.mark.parametrize( + "value, fmt, expected", + [ + (dt.date(2009, 1, 1), "%Y-%m-%d", "2009-01-01"), + (dt.date(2009, 1, 1), "%d/%m/%Y", "01/01/2009"), + (dt.datetime(2009, 1, 1, 6, 0), "%Y-%m-%d", "2009-01-01"), + ], + ids=["date", "custom-fmt", "timestamp"], + ) + def test_a_date_yaml_already_parsed_is_written_back_in_fmt(self, value, fmt, expected): + """Test that an unquoted YAML date is accepted and rendered in the block's format. + + Args: + value: What YAML hands pydantic for an unquoted date or timestamp. + fmt: The block's date format. + expected: The string the field should end up holding. + + Test scenario: + `start: 2009-01-01` written without quotes is a `datetime.date` by the time the + schema sees it, and the field is a string because the constructor parses it with + `fmt`. Rejecting it would reject the spelling a YAML author writes first, over a + difference that carries no information. + """ + config = CatchmentConfig(name="Coello", start=value, end=value, fmt=fmt) + + assert config.start == expected, f"expected {expected!r}, got {config.start!r}" + assert isinstance(config.start, str), ( + f"the field must still hold a string, got {type(config.start)}" + ) + + def test_a_quoted_date_is_left_exactly_as_written(self): + """Test that the normaliser does not touch a date that is already text. + + Test scenario: + A string may be in any format the author declares, including ones no `date` + round-trips through, so it has to pass through untouched. + """ + config = CatchmentConfig( + name="Coello", start="01/01/2009", end="31/12/2011", fmt="%d/%m/%Y" + ) + + assert config.start == "01/01/2009", f"start was rewritten: {config.start}" + + @pytest.mark.parametrize( + "values, why", + [ + ("not-a-mapping", "a scalar block"), + ({"name": "Coello", "start": "2009-01-01", "end": "2009-01-10", "fmt": 5}, "a non-string fmt"), + ], + ids=["scalar", "non-string-fmt"], + ) + def test_the_normaliser_defers_to_pydantic_for_what_it_cannot_render(self, values, why): + """Test that unrenderable input is passed through for pydantic to report. + + Args: + values: Raw input the normaliser cannot act on. + why: What is wrong with it, for the failure message. + + Test scenario: + The normaliser runs before validation, so it sees raw input. Raising its own + error for a malformed block would replace pydantic's precise, field-located + message with a vaguer one. + """ + with pytest.raises(ValidationError): + CatchmentConfig.model_validate(values) + class TestMeteoConfig: """Tests for the `meteo` block.""" @@ -664,6 +731,268 @@ def test_parameters_and_gauges_may_both_be_omitted(self, distributed_mapping): assert config.parameters is None, "parameters should be absent, not defaulted" assert config.gauges is None, "gauges should be absent, not defaulted" + @pytest.mark.parametrize( + "block, field, value", + [ + ("gauges", "table", "gauges.csv"), + ("gauges", "column", "name"), + ("meteo", "precipitation", "prec"), + ("meteo", "start", "2009-01-01"), + ("meteo", "end", "2011-12-31"), + ("meteo", "glob", "*.tif"), + ], + ids=["table", "column", "driver", "window-start", "window-end", "glob"], + ) + def test_a_lumped_configuration_refuses_a_field_it_would_not_read( + self, lumped_mapping, block, field, value + ): + """Test that each field a lumped run never reads is named and refused. + + Args: + lumped_mapping: A complete lumped configuration. + block: Which block the field lives in. + field: The inapplicable field. + value: A plausible value for it. + + Test scenario: + A lumped run reads one CSV of catchment-average drivers and locates no gauges, + so a grid driver, a reader knob, a window or a gauge table is a line with no + effect. Dropping it silently is the failure `extra="forbid"` exists to prevent, + arrived at by another route. + """ + lumped_mapping[block][field] = value + + with pytest.raises(ValidationError, match="read by nothing") as exc: + RunConfig.model_validate(lumped_mapping) + + assert f"{block}.{field}" in str(exc.value), ( + f"the error should name the offending field: {exc.value}" + ) + + @pytest.mark.parametrize( + "source, patch, refused", + [ + ("netcdf", {"path": "m.nc", "glob": "*.tif"}, "meteo.glob"), + ("netcdf", {"path": "m.nc", "per_variable": {"p": {}}}, "meteo.per_variable"), + ("netcdf", {"path": "m.nc", "variable": "pre"}, "meteo.variable"), + ("netcdf_files", {"path": "m.nc"}, "meteo.path"), + ("rasters", {"path": "m.nc"}, "meteo.path"), + ("rasters", {"variable": "pre"}, "meteo.variable"), + ], + ids=[ + "netcdf-glob", + "netcdf-per-variable", + "netcdf-variable", + "netcdf-files-path", + "rasters-path", + "rasters-variable", + ], + ) + def test_a_source_refuses_the_fields_its_own_branch_never_reads( + self, distributed_mapping, source, patch, refused + ): + """Test that each `meteo.source` refuses the knobs belonging to the other two. + + Args: + distributed_mapping: A complete distributed configuration. + source: The source under test. + patch: Fields to set on the `meteo` block, including the inapplicable one. + refused: The field the error must name. + + Test scenario: + The three branches of `MeteoInputs.from_config` read different fields: the + raster reader takes `glob` and `per_variable` and no `path`, `netcdf` takes a + `path` and no reader knobs, `netcdf_files` takes a `variable`. Setting one + outside its source's set says something the run will not do. + """ + distributed_mapping["meteo"]["source"] = source + distributed_mapping["meteo"].update(patch) + + with pytest.raises(ValidationError, match="read by nothing") as exc: + RunConfig.model_validate(distributed_mapping) + + assert refused in str(exc.value), f"the error should name {refused}: {exc.value}" + + def test_a_default_the_author_never_wrote_is_not_refused(self, lumped_mapping): + """Test that only explicitly written fields count as inapplicable. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + Every refused field has a default, so testing the value rather than whether it + was set would reject every lumped configuration ever written -- `meteo.glob` + alone defaults to `"*.tif"`. The check reads `model_fields_set`. + """ + config = RunConfig.model_validate(lumped_mapping) + + assert config.meteo.glob == "*.tif", ( + f"the default should still be there, got {config.meteo.glob}" + ) + + @pytest.mark.parametrize( + "maxbas, expected", + [(True, "maxbas"), (False, "muskingum")], + ids=["maxbas-set", "muskingum-set"], + ) + def test_an_unstated_routing_method_is_derived_from_the_parameter_set( + self, lumped_mapping, maxbas, expected + ): + """Test that `routing_method` follows `parameters.maxbas` when it is not written. + + Args: + lumped_mapping: A complete lumped configuration. + maxbas: What the parameter set carries. + expected: The routing method that should be derived from it. + + Test scenario: + The two describe the same choice from opposite sides. Left at its `muskingum` + default, a MAXBAS run carried a `routing_method` contradicting what it does -- + and that attribute is what `distrrm.SpatialRouting` keys off. + """ + lumped_mapping["parameters"]["maxbas"] = maxbas + assert "routing_method" not in lumped_mapping["catchment"], ( + "this test is about the unstated case" + ) + + config = RunConfig.model_validate(lumped_mapping) + + assert config.catchment.routing_method == expected, ( + f"expected {expected!r} derived from maxbas={maxbas}, got " + f"{config.catchment.routing_method!r}" + ) + + def test_a_lumped_routing_method_must_still_agree_with_the_parameter_set( + self, lumped_mapping + ): + """Test that the agreement check is no longer scoped to distributed runs. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + The parameter-count check cannot catch this: the two counts differ by one and + `maxbas` is what selects which is expected, so a contradicting pair still counts + correctly and then reads the wrong parameter as the routing one. + """ + lumped_mapping["catchment"]["routing_method"] = "muskingum" + lumped_mapping["parameters"]["maxbas"] = True + + with pytest.raises(ValidationError, match="must agree"): + RunConfig.model_validate(lumped_mapping) + + def test_the_derivation_runs_before_the_flow_direction_check(self, distributed_mapping): + """Test that a derived MAXBAS run may omit the flow-direction raster. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + MAXBAS sends every cell straight to the outlet and never reads a flow direction, + but that requirement is keyed off `routing_method` -- so if the block checks ran + before the derivation, the still-defaulted `muskingum` would demand a raster the + run has no use for. + """ + distributed_mapping["parameters"]["maxbas"] = True + del distributed_mapping["flow_network"]["flow_direction"] + distributed_mapping["catchment"].pop("routing_method", None) + + config = RunConfig.model_validate(distributed_mapping) + + assert config.catchment.routing_method == "maxbas", ( + f"expected the derived method, got {config.catchment.routing_method!r}" + ) + + @pytest.mark.parametrize( + "window", + [ + {"start": "2012-06-01"}, + {"end": "2008-01-01"}, + {"start": "2010-01-01", "end": "2009-01-01"}, + ], + ids=["start-after-catchment-end", "end-before-catchment-start", "both-inverted"], + ) + def test_the_resolved_meteorological_window_must_run_forwards( + self, distributed_mapping, window + ): + """Test that the window the run will use is checked, not the two literal pairs. + + Args: + distributed_mapping: A complete distributed configuration. + window: A `meteo` window that inverts the effective period. + + Test scenario: + `MeteoInputs.from_config` takes each bound from `meteo` when stated and falls + back to `catchment` otherwise, so a block stating only one half can invert the + effective window while neither pair is inverted on its own. + """ + distributed_mapping["meteo"].update(window) + + with pytest.raises(ValidationError, match="ends before it starts"): + RunConfig.model_validate(distributed_mapping) + + def test_a_meteo_window_inside_the_catchment_period_is_accepted( + self, distributed_mapping + ): + """Test that a genuine sub-period is not caught by the window check. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + Narrowing the drivers to part of the catchment's period is the reason the two + `meteo` bounds exist, so the stricter check must not cost that. + """ + distributed_mapping["meteo"]["start"] = "2009-01-02" + distributed_mapping["meteo"]["end"] = "2009-01-05" + + config = RunConfig.model_validate(distributed_mapping) + + assert config.meteo.start == "2009-01-02", ( + f"the stated window should survive: {config.meteo}" + ) + + def test_the_gauge_table_format_falls_back_to_the_discharge_format( + self, distributed_mapping + ): + """Test that `table_fmt` defaults to `fmt` rather than to a format of its own. + + Args: + distributed_mapping: A complete distributed configuration. + + Test scenario: + The two parse different files -- the table's validity-period columns and the + discharge CSV index -- but one hand usually writes both, so the common case + should stay a single field. + """ + distributed_mapping["gauges"]["fmt"] = "%d/%m/%Y" + + config = RunConfig.model_validate(distributed_mapping) + + assert config.gauges.table_fmt is None, ( + f"table_fmt should stay unset so the builder can fall back: " + f"{config.gauges.table_fmt}" + ) + assert config.gauges.fmt == "%d/%m/%Y", "the discharge format should be kept" + + + def test_a_lumped_configuration_may_omit_gauges(self, lumped_mapping): + """Test that a lumped run with no gauges block validates. + + Args: + lumped_mapping: A complete lumped configuration. + + Test scenario: + The inapplicable-field check for `gauges` only runs when the block is present, so + a lumped run that is not scored against observations has to pass through it + untouched rather than tripping on a block that is not there. + """ + del lumped_mapping["gauges"] + + config = RunConfig.model_validate(lumped_mapping) + + assert config.gauges is None, "gauges should be absent, not defaulted" + class TestMeteoInputsFromConfig: """Tests for the `meteo.source` dispatch in `MeteoInputs.from_config`.""" @@ -803,6 +1132,40 @@ def test_the_netcdf_source_needs_a_path(self): ) + def test_the_netcdf_source_reads_the_three_variables_from_one_file( + self, coello_start_date: str, coello_end_date: str + ): + """Test that `source="netcdf"` builds a grid from one file's three variables. + + Args: + coello_start_date: Simulation start date. + coello_end_date: Simulation end date. + + Test scenario: + The other two sources have direct success tests; this branch was covered only + transitively through `from_yaml`, so a change to how `meteo.path` or the variable + names are forwarded would have surfaced only there. + """ + config = MeteoConfig( + source="netcdf", + path=COMBINED_NC, + precipitation="precipitation", + temperature="temperature", + evapotranspiration="evapotranspiration", + ) + + meteo = MeteoInputs.from_config( + config, start=coello_start_date, end="2009-01-10" + ) + + assert meteo.precipitation.shape == meteo.temperature.shape, ( + f"the three cubes must agree: {meteo.shape}" + ) + assert meteo.time_steps == 10, ( + f"the window should hold ten steps, got {meteo.time_steps}" + ) + + class TestRoutingMethodNormalisation: """Tests for the `routing_method` canonicalisation in `Catchment.__init__`.""" @@ -876,6 +1239,39 @@ def test_an_unknown_routing_method_is_refused(self): ) + @pytest.mark.parametrize( + "argument", + ["spatial_resolution", "temporal_resolution", "routing_method"], + ) + def test_a_mode_argument_that_is_not_a_string_names_itself(self, argument): + """Test that each mode argument reports its own name when handed a non-string. + + Args: + argument: The constructor argument under test. + + Test scenario: + All three are lower-cased, so a non-string used to reach `.lower()` and raise an + `AttributeError` naming neither the argument nor the class. `Calibration` made + that reachable through its own signature, which declared all three `str | None`. + """ + with pytest.raises(TypeError, match=f"{argument} must be a string") as exc: + Catchment("Coello", "2009-01-01", "2009-01-10", **{argument: None}) + + assert "NoneType" in str(exc.value), ( + f"the error should name what it got: {exc.value}" + ) + + def test_calibration_accepts_the_same_three_arguments(self): + """Test that the guard reaches `Calibration`, which passes the arguments down. + + Test scenario: + `Calibration.__init__` forwards all three to `Catchment.__init__` unchanged, and + its annotations used to advertise a `None` that crashed. + """ + with pytest.raises(TypeError, match="routing_method must be a string"): + Calibration("Coello", "2009-01-01", "2009-01-10", routing_method=None) + + class TestCatchmentFromYaml: """Tests for `Catchment.from_yaml`, which turns a configuration into a built model.""" @@ -1404,3 +1800,240 @@ def test_the_configuration_is_read_from_the_given_path( assert Catchment.from_yaml(first).name == "Coello" assert Catchment.from_yaml(second).name == "Elsewhere" + + def test_a_missing_file_names_the_path(self, tmp_path): + """Test that a configuration path that does not exist raises `FileNotFoundError`. + + Args: + tmp_path: pytest temporary directory. + + Test scenario: + Documented in `Raises` but untested. The path is opened directly, so the error + comes from `read_text` and carries the name. + """ + missing = tmp_path / "not-here.yaml" + + with pytest.raises(FileNotFoundError): + Catchment.from_yaml(str(missing)) + + def test_malformed_yaml_is_reported_as_malformed(self, tmp_path): + """Test that a file YAML cannot parse raises `yaml.YAMLError`. + + Args: + tmp_path: pytest temporary directory. + + Test scenario: + Also documented and untested. An unclosed bracket is a parse error, which has to + surface as one rather than as a validation error about a missing block. + """ + path = tmp_path / "broken.yaml" + path.write_text("catchment: {name: Coello\n", encoding="utf-8") + + with pytest.raises(yaml.YAMLError): + Catchment.from_yaml(str(path)) + + def test_a_top_level_scalar_is_refused(self, tmp_path): + """Test that a file holding a bare scalar is refused rather than indexed. + + Args: + tmp_path: pytest temporary directory. + + Test scenario: + A one-word file parses to a string, not to None, so it bypasses the empty-file + message and reaches pydantic. Pins that it fails there rather than raising an + `AttributeError` somewhere in the build. + """ + path = tmp_path / "scalar.yaml" + path.write_text("hello\n", encoding="utf-8") + + with pytest.raises(ValidationError, match="valid dictionary"): + Catchment.from_yaml(str(path)) + + def test_every_missing_input_path_is_named_at_once(self, distributed_mapping, tmp_path): + """Test that the pre-flight check reports all the missing paths together. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The readers fail one at a time and in the order the build calls them, so a typo + in the gauge table used to be reported only after the whole meteorological cube + and the parameter folder had been read. Two typos should now be reported in one + message, before anything is opened. + """ + distributed_mapping["parameters"]["path"] = "no/such/parameters" + distributed_mapping["gauges"]["table"] = "no/such/gauges.csv" + + with pytest.raises(FileNotFoundError) as exc: + Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + message = str(exc.value) + assert "parameters.path" in message and "gauges.table" in message, ( + f"both missing paths should be named in one message: {message}" + ) + + def test_a_netcdf_variable_name_is_not_checked_for_existence( + self, distributed_mapping, tmp_path + ): + """Test that the pre-flight check does not treat a variable name as a path. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + Under `source: netcdf` the three driver fields name variables inside + `meteo.path`, so checking them as paths would fail every NetCDF configuration. + The fixture already uses that source. + """ + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo is not None, "the build should have reached the readers" + + def test_the_gauge_columns_come_from_the_configured_column( + self, distributed_mapping, tmp_path + ): + """Test that `gauges.column` labels the hydrograph frame and every column is filled. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + The frame is labelled from `column` but each file is named after `id`. Filling by + `int(name)` instead of by the label left a `column != "id"` table with the + requested columns all-NaN and a second, id-named set beside them -- silently, and + the metrics were then computed over both. + """ + distributed_mapping["gauges"]["column"] = "name" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + columns = list(model.QGauges.columns) + assert all(isinstance(name, str) for name in columns), ( + f"the frame should carry the table's names, got {columns}" + ) + all_nan = [name for name in columns if model.QGauges[name].isna().all()] + assert not all_nan, f"no column should be left unfilled, got {all_nan}" + + def test_the_gauge_table_format_can_differ_from_the_discharge_one( + self, distributed_mapping, tmp_path + ): + """Test that `table_fmt` is what reaches `read_gauge_table`. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + One field used to feed both readers, so the two files could not disagree. The + Coello table carries no `start` / `end` columns, so an unused `table_fmt` must + simply be accepted and forwarded rather than applied to the discharge index. + """ + distributed_mapping["gauges"]["table_fmt"] = "%d/%m/%Y" + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.config.gauges.table_fmt == "%d/%m/%Y", ( + f"the table format should be kept on the config: {model.config.gauges}" + ) + assert not model.QGauges.isna().all().all(), ( + "the discharge index must still be parsed with gauges.fmt" + ) + + def test_an_unquoted_date_builds_the_same_model_as_a_quoted_one( + self, distributed_mapping, tmp_path + ): + """Test the unquoted-date path end to end, not only at the schema. + + Args: + distributed_mapping: A complete distributed configuration. + tmp_path: pytest temporary directory. + + Test scenario: + `start: 2009-01-01` without quotes is what a YAML author writes first, and it + reaches pydantic as a `date`. The model built from it must be identical to the + one built from the quoted spelling. + """ + quoted = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + unquoted_mapping = copy.deepcopy(distributed_mapping) + unquoted_mapping["catchment"]["start"] = dt.date.fromisoformat( + distributed_mapping["catchment"]["start"] + ) + other = tmp_path / "unquoted" + other.mkdir() + unquoted = Catchment.from_yaml(write_yaml(unquoted_mapping, other)) + + assert unquoted.start == quoted.start, ( + f"expected {quoted.start}, got {unquoted.start}" + ) + assert len(unquoted.date_index) == len(quoted.date_index), ( + "both spellings should span the same period" + ) + + def test_a_raster_source_configuration_reads_the_three_folders( + self, + distributed_mapping, + coello_prec_path: str, + coello_temp_path: str, + coello_evap_path: str, + tmp_path, + ): + """Test the raster branch of the build, including its path check. + + Args: + distributed_mapping: A complete distributed configuration. + coello_prec_path: Folder of precipitation rasters. + coello_temp_path: Folder of temperature rasters. + coello_evap_path: Folder of evapotranspiration rasters. + tmp_path: pytest temporary directory. + + Test scenario: + The fixture drives every other `from_yaml` test from one combined NetCDF, where + the three driver fields are variable names. Under `rasters` they are folders that + the pre-flight check does look for, and `MeteoInputs.from_rasters` is a different + loader -- so both are covered only here. + """ + distributed_mapping["meteo"] = { + "source": "rasters", + "precipitation": str(Path(coello_prec_path).resolve()), + "temperature": str(Path(coello_temp_path).resolve()), + "evapotranspiration": str(Path(coello_evap_path).resolve()), + "file_name_data_fmt": "%Y.%m.%d", + } + + model = Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + + assert model.meteo.time_steps == len(model.date_index), ( + f"the drivers must span the model period: {model.meteo.time_steps} against " + f"{len(model.date_index)}" + ) + + def test_a_missing_driver_folder_is_reported_before_anything_is_read( + self, distributed_mapping, coello_temp_path: str, coello_evap_path: str, tmp_path + ): + """Test that a raster driver folder is checked for existence like any other path. + + Args: + distributed_mapping: A complete distributed configuration. + coello_temp_path: Folder of temperature rasters. + coello_evap_path: Folder of evapotranspiration rasters. + tmp_path: pytest temporary directory. + + Test scenario: + A misspelled folder used to be reported from inside pyramids, after the reader + had opened whatever it could. Under a NetCDF source the driver fields are + variable names and must not be checked; under this one they are paths and must + be. + """ + distributed_mapping["meteo"] = { + "source": "rasters", + "precipitation": str(tmp_path / "no-such-folder"), + "temperature": str(Path(coello_temp_path).resolve()), + "evapotranspiration": str(Path(coello_evap_path).resolve()), + } + + with pytest.raises(FileNotFoundError, match="meteo.precipitation"): + Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) diff --git a/tests/rrm/catchment/test_maxbas_routing_variants.py b/tests/rrm/catchment/test_maxbas_routing_variants.py index 693798ca..9e83bd6a 100644 --- a/tests/rrm/catchment/test_maxbas_routing_variants.py +++ b/tests/rrm/catchment/test_maxbas_routing_variants.py @@ -286,3 +286,68 @@ def test_routing_without_a_function_is_rejected( with pytest.raises(ValueError, match="routing_fn"): Run.runLumped(model, Route=1) + + +class TestCalculateWeightsGuard: + """Tests for the MAXBAS lower bound in `Routing.calculate_weights`.""" + + @pytest.mark.parametrize( + "maxbas, symptom", + [ + (0.5, "one weight and an all-zero hydrograph, silently"), + (0, "an IndexError about axis 1"), + (float("nan"), "'cannot convert float NaN to integer'"), + ], + ids=["fractional", "zero", "nan"], + ) + def test_a_maxbas_below_one_is_refused(self, maxbas, symptom): + """Test that each below-one MAXBAS is refused by name. + + Args: + maxbas: A MAXBAS value below the one whole step a triangle needs. + symptom: What it produced before the guard, for the failure message. + + Test scenario: + `triangular_routing_2` and the three conceptual models already carried this + check; `calculate_weights` -- which `triangular_routing_1` and `DistMaxbas1` + resolve their weights through -- did not, and each value failed differently or + not at all. + """ + with pytest.raises(ValueError, match="at least 1") as exc: + Routing.calculate_weights(maxbas) + + assert str(maxbas) in str(exc.value), ( + f"the error should name the value, given it used to produce {symptom}: {exc.value}" + ) + + def test_the_guard_reaches_triangular_routing_1(self): + """Test that the routing function itself refuses, not only the weights helper. + + Test scenario: + `triangular_routing_1` is what the lumped MAXBAS example and `DistMaxbas1` call, + and it delegates to `calculate_weights` on its first line -- so the guard has to + surface there rather than being swallowed. + """ + q = np.array([0.0, 1.0, 3.0, 7.0, 10.0]) + + with pytest.raises(ValueError, match="at least 1"): + Routing.triangular_routing_1(q, 0.2) + + @pytest.mark.parametrize("maxbas", [1, 4.5, 5], ids=["minimum", "fractional", "whole"]) + def test_a_valid_maxbas_still_routes(self, maxbas): + """Test that the guard leaves every accepted MAXBAS working. + + Args: + maxbas: A MAXBAS at or above the bound, integer and fractional. + + Test scenario: + The bound is `not maxbas >= 1` so that NaN fails it; the boundary value itself + must still pass, and fractional support is what separates this function from + `triangular_routing_2`. + """ + weights = Routing.calculate_weights(maxbas) + + assert len(weights) >= 1, f"expected weights for maxbas={maxbas}, got {weights}" + assert np.isclose(weights.sum(), 1.0, atol=0.05), ( + f"triangular weights should be normalised, got {weights.sum()}" + ) diff --git a/tests/rrm/catchment/test_save_results_distributed.py b/tests/rrm/catchment/test_save_results_distributed.py index f926ffc4..1df6e380 100644 --- a/tests/rrm/catchment/test_save_results_distributed.py +++ b/tests/rrm/catchment/test_save_results_distributed.py @@ -128,3 +128,84 @@ def test_save_results_distributed_values_match_the_model_array( np.testing.assert_allclose( actual, expected, rtol=1e-5, err_msg="the first raster must hold the first step" ) + + +def test_save_results_joins_a_directory_written_without_a_separator( + coello_run: Catchment, coello_acc_path: str, tmp_path +): + """Test that a directory given without a trailing separator still writes inside it. + + Args: + coello_run: Distributed Coello catchment with a completed run. + coello_acc_path: Path to the flow-accumulation raster used as the template. + tmp_path: Parent of the destination directory. + + Test scenario: + The names used to be built by concatenation, so `some/dir` produced + `some/dirResult_2009-01-01.tif` -- a sibling of the directory rather than a file in + it. The other tests in this file all pass a trailing separator, so none of them + would notice. + """ + out = tmp_path / "no-separator" + out.mkdir() + + coello_run.save_results( + flow_acc_path=coello_acc_path, + result=4, + start="2009-01-01", + end="2009-01-02", + path=str(out), + ) + + assert len(sorted(out.glob("*.tif"))) == 2, ( + f"the rasters must land inside the directory, found {sorted(tmp_path.iterdir())}" + ) + + +def test_save_results_creates_the_directory_it_is_given( + coello_run: Catchment, coello_acc_path: str, tmp_path +): + """Test that a destination directory that does not exist yet is created. + + Args: + coello_run: Distributed Coello catchment with a completed run. + coello_acc_path: Path to the flow-accumulation raster used as the template. + tmp_path: Parent of the destination directory. + + Test scenario: + `outputs.results_dir` in a run configuration names where results go, and nothing + guarantees it exists before the first run. Covers a nested path, so a single + `mkdir` would not be enough. + """ + out = tmp_path / "nested" / "results" + + coello_run.save_results( + flow_acc_path=coello_acc_path, + result=4, + start="2009-01-01", + end="2009-01-02", + path=str(out), + ) + + assert len(sorted(out.glob("*.tif"))) == 2, ( + f"the directory must be created and written into, got {out.exists()}" + ) + + +def test_save_results_refuses_a_path_that_is_not_a_string(coello_run: Catchment): + """Test that a non-string `path` is refused by name rather than by concatenation. + + Args: + coello_run: Distributed Coello catchment with a completed run. + + Test scenario: + `outputs.results_dir` is optional in a run configuration, so a caller forwarding it + straight through can hold None. That used to surface as a `TypeError` from a string + concatenation, naming neither the argument nor what it should be. + """ + with pytest.raises(TypeError, match="path must be a string") as exc: + coello_run.save_results(flow_acc_path="unused", result=1, path=None) + + assert "NoneType" in str(exc.value), ( + f"the error should name what it got: {exc.value}" + ) From 538373e43c8026ba405ad0dd96f10de9c5740567 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 30 Aug 2026 23:58:48 +0200 Subject: [PATCH 54/61] docs(run): add executable examples to the two symbols that lacked them Of the public symbols this round touched, `Run.from_yaml` and `config.missing_drivers_message` were the two carrying no Examples section. Both now have one, and both run: the refusal example prints the first clause of the message it raises, and the one beside it builds a model from a shipped configuration and routes it, which is the pattern the refusal is pointing at. The message helper shows the singular and plural forms it chooses between. `run.py` joins the `doctests` task now that its examples execute -- 26 passing across the five modules. --- pyproject.toml | 2 +- src/hapi/config.py | 17 +++++++++++++++++ src/hapi/run.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 95c754e5..0b78ed0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -262,7 +262,7 @@ description = "Run all test suite" cmd = [ "pytest", "--doctest-modules", "-p", "no:cacheprovider", "--no-cov", "src/hapi/config.py", "src/hapi/catchment.py", - "src/hapi/inputs.py", "src/hapi/routing.py", + "src/hapi/inputs.py", "src/hapi/routing.py", "src/hapi/run.py", ] description = "Run the doctests of the modules whose examples are executable" diff --git a/src/hapi/config.py b/src/hapi/config.py index d55ab284..36928d62 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -184,6 +184,23 @@ def missing_drivers_message(missing: Sequence[str]) -> str: Returns: str: The message. + + Examples: + - One driver missing reads as a singular: + ```python + >>> from hapi.config import missing_drivers_message + >>> missing_drivers_message(["temperature"]) + 'a distributed run needs all three meteorological drivers; temperature is unset' + + ``` + - Several are listed in the order they are given: + ```python + >>> from hapi.config import missing_drivers_message + >>> message = missing_drivers_message(["temperature", "evapotranspiration"]) + >>> message.split("; ")[1] + 'temperature, evapotranspiration are unset' + + ``` """ return ( f"a distributed run needs all three meteorological drivers; " diff --git a/src/hapi/run.py b/src/hapi/run.py index 43f14d62..07f07ef0 100644 --- a/src/hapi/run.py +++ b/src/hapi/run.py @@ -101,6 +101,34 @@ def from_yaml(cls, path: str | Path) -> NoReturn: Raises: TypeError: Always. + + Examples: + - The refusal names the pattern to use instead: + ```python + >>> from hapi.run import Run + >>> try: + ... Run.from_yaml("coello-lumped-model-run.yaml") + ... except TypeError as error: + ... print(str(error).split(";")[0]) + Run cannot be built from a configuration + + ``` + - Build the model with `Catchment.from_yaml` and hand it to the entry point: + ```python + >>> from hapi.catchment import Catchment + >>> from hapi.routing import Routing + >>> from hapi.run import Run + >>> model = Catchment.from_yaml( + ... "examples/hydrological-model/coello/run/coello-lumped-model-run.yaml" + ... ) + >>> Run.runLumped(model, 1, Routing.muskingum_v) + >>> len(model.Qsim) + 1095 + + ``` + + See Also: + hapi.catchment.Catchment.from_yaml: The classmethod that does build a model. """ raise TypeError( "Run cannot be built from a configuration; it holds the entry points that run a " From 01de124745deb93cb3cc7eb57d8ff48b2252602f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:05:43 +0200 Subject: [PATCH 55/61] style: apply ruff-format and normalise line endings The pre-commit job failed on the round-2 commits: ruff-format wraps at 88 characters, which is narrower than the 120 the prose limit allows, so several test signatures and one `parametrize` call needed breaking. Three files had also picked up mixed line endings. Formatting only -- the suite and the doctests are unchanged at 546 and 26. --- src/hapi/catchment.py | 5 +- src/hapi/config.py | 4 +- tests/rrm/catchment/test_config.py | 59 ++++++++++++++----- .../catchment/test_maxbas_routing_variants.py | 4 +- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index d674306c..0fd5ead7 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -160,7 +160,10 @@ def _check_the_configured_paths_exist(config: RunConfig, distributed: bool) -> N ] if config.flow_network is not None: candidates += [ - ("flow_network.flow_accumulation", config.flow_network.flow_accumulation), + ( + "flow_network.flow_accumulation", + config.flow_network.flow_accumulation, + ), ("flow_network.flow_direction", config.flow_network.flow_direction), ] else: diff --git a/src/hapi/config.py b/src/hapi/config.py index 36928d62..99cc58d5 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -521,7 +521,9 @@ def _check_the_routing_method_matches_the_parameter_set(self) -> RunConfig: return self if "routing_method" not in self.catchment.model_fields_set: - self.catchment.routing_method = "maxbas" if self.parameters.maxbas else "muskingum" + self.catchment.routing_method = ( + "maxbas" if self.parameters.maxbas else "muskingum" + ) return self if (self.catchment.routing_method == "maxbas") != self.parameters.maxbas: diff --git a/tests/rrm/catchment/test_config.py b/tests/rrm/catchment/test_config.py index 98e9f1ba..b6a4305d 100644 --- a/tests/rrm/catchment/test_config.py +++ b/tests/rrm/catchment/test_config.py @@ -237,7 +237,9 @@ def test_dates_stay_strings(self): ], ids=["date", "custom-fmt", "timestamp"], ) - def test_a_date_yaml_already_parsed_is_written_back_in_fmt(self, value, fmt, expected): + def test_a_date_yaml_already_parsed_is_written_back_in_fmt( + self, value, fmt, expected + ): """Test that an unquoted YAML date is accepted and rendered in the block's format. Args: @@ -275,11 +277,21 @@ def test_a_quoted_date_is_left_exactly_as_written(self): "values, why", [ ("not-a-mapping", "a scalar block"), - ({"name": "Coello", "start": "2009-01-01", "end": "2009-01-10", "fmt": 5}, "a non-string fmt"), + ( + { + "name": "Coello", + "start": "2009-01-01", + "end": "2009-01-10", + "fmt": 5, + }, + "a non-string fmt", + ), ], ids=["scalar", "non-string-fmt"], ) - def test_the_normaliser_defers_to_pydantic_for_what_it_cannot_render(self, values, why): + def test_the_normaliser_defers_to_pydantic_for_what_it_cannot_render( + self, values, why + ): """Test that unrenderable input is passed through for pydantic to report. Args: @@ -598,7 +610,9 @@ def test_distributed_requires_all_three_drivers(self, distributed_mapping, drive """ del distributed_mapping["meteo"][driver] - with pytest.raises(ValidationError, match="all three meteorological drivers") as exc: + with pytest.raises( + ValidationError, match="all three meteorological drivers" + ) as exc: RunConfig.model_validate(distributed_mapping) assert driver in str(exc.value), ( @@ -773,7 +787,11 @@ def test_a_lumped_configuration_refuses_a_field_it_would_not_read( "source, patch, refused", [ ("netcdf", {"path": "m.nc", "glob": "*.tif"}, "meteo.glob"), - ("netcdf", {"path": "m.nc", "per_variable": {"p": {}}}, "meteo.per_variable"), + ( + "netcdf", + {"path": "m.nc", "per_variable": {"p": {}}}, + "meteo.per_variable", + ), ("netcdf", {"path": "m.nc", "variable": "pre"}, "meteo.variable"), ("netcdf_files", {"path": "m.nc"}, "meteo.path"), ("rasters", {"path": "m.nc"}, "meteo.path"), @@ -811,7 +829,9 @@ def test_a_source_refuses_the_fields_its_own_branch_never_reads( with pytest.raises(ValidationError, match="read by nothing") as exc: RunConfig.model_validate(distributed_mapping) - assert refused in str(exc.value), f"the error should name {refused}: {exc.value}" + assert refused in str(exc.value), ( + f"the error should name {refused}: {exc.value}" + ) def test_a_default_the_author_never_wrote_is_not_refused(self, lumped_mapping): """Test that only explicitly written fields count as inapplicable. @@ -881,7 +901,9 @@ def test_a_lumped_routing_method_must_still_agree_with_the_parameter_set( with pytest.raises(ValidationError, match="must agree"): RunConfig.model_validate(lumped_mapping) - def test_the_derivation_runs_before_the_flow_direction_check(self, distributed_mapping): + def test_the_derivation_runs_before_the_flow_direction_check( + self, distributed_mapping + ): """Test that a derived MAXBAS run may omit the flow-direction raster. Args: @@ -910,7 +932,11 @@ def test_the_derivation_runs_before_the_flow_direction_check(self, distributed_m {"end": "2008-01-01"}, {"start": "2010-01-01", "end": "2009-01-01"}, ], - ids=["start-after-catchment-end", "end-before-catchment-start", "both-inverted"], + ids=[ + "start-after-catchment-end", + "end-before-catchment-start", + "both-inverted", + ], ) def test_the_resolved_meteorological_window_must_run_forwards( self, distributed_mapping, window @@ -975,7 +1001,6 @@ def test_the_gauge_table_format_falls_back_to_the_discharge_format( ) assert config.gauges.fmt == "%d/%m/%Y", "the discharge format should be kept" - def test_a_lumped_configuration_may_omit_gauges(self, lumped_mapping): """Test that a lumped run with no gauges block validates. @@ -1131,7 +1156,6 @@ def test_the_netcdf_source_needs_a_path(self): ) ) - def test_the_netcdf_source_reads_the_three_variables_from_one_file( self, coello_start_date: str, coello_end_date: str ): @@ -1238,7 +1262,6 @@ def test_an_unknown_routing_method_is_refused(self): f"the error should echo the value given: {exc.value}" ) - @pytest.mark.parametrize( "argument", ["spatial_resolution", "temporal_resolution", "routing_method"], @@ -1697,7 +1720,9 @@ def test_relative_paths_resolve_against_the_configuration_file( monkeypatch.chdir(tmp_path) model = Catchment.from_yaml(str(path)) - assert model.meteo is not None, "the drivers should resolve from the config's own dir" + assert model.meteo is not None, ( + "the drivers should resolve from the config's own dir" + ) assert model.flow_network is not None, "the network should resolve too" def test_a_netcdf_variable_name_is_not_treated_as_a_path( @@ -1849,7 +1874,9 @@ def test_a_top_level_scalar_is_refused(self, tmp_path): with pytest.raises(ValidationError, match="valid dictionary"): Catchment.from_yaml(str(path)) - def test_every_missing_input_path_is_named_at_once(self, distributed_mapping, tmp_path): + def test_every_missing_input_path_is_named_at_once( + self, distributed_mapping, tmp_path + ): """Test that the pre-flight check reports all the missing paths together. Args: @@ -2012,7 +2039,11 @@ def test_a_raster_source_configuration_reads_the_three_folders( ) def test_a_missing_driver_folder_is_reported_before_anything_is_read( - self, distributed_mapping, coello_temp_path: str, coello_evap_path: str, tmp_path + self, + distributed_mapping, + coello_temp_path: str, + coello_evap_path: str, + tmp_path, ): """Test that a raster driver folder is checked for existence like any other path. diff --git a/tests/rrm/catchment/test_maxbas_routing_variants.py b/tests/rrm/catchment/test_maxbas_routing_variants.py index 9e83bd6a..07275a9a 100644 --- a/tests/rrm/catchment/test_maxbas_routing_variants.py +++ b/tests/rrm/catchment/test_maxbas_routing_variants.py @@ -333,7 +333,9 @@ def test_the_guard_reaches_triangular_routing_1(self): with pytest.raises(ValueError, match="at least 1"): Routing.triangular_routing_1(q, 0.2) - @pytest.mark.parametrize("maxbas", [1, 4.5, 5], ids=["minimum", "fractional", "whole"]) + @pytest.mark.parametrize( + "maxbas", [1, 4.5, 5], ids=["minimum", "fractional", "whole"] + ) def test_a_valid_maxbas_still_routes(self, maxbas): """Test that the guard leaves every accepted MAXBAS working. From 1b42b5e7cf3c33599db33f0a541df3c508ba55d2 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:09:05 +0200 Subject: [PATCH 56/61] fix(sonar): repair the two defects the PR analysis found `read_lumped_model` measured `initial_condition` before typing it, so `None` reported "object of type 'NoneType' has no len()" instead of naming the argument -- and the type check that followed carried an `is not None` that could never be false, because `len` would already have raised (S2589). Typed first, then measured; a non-list is refused the same way it was, with the message it had. `lumpedCalibration` indexed `initial_values[i]` over `range(len(self.LB))` behind a guard that only checked the list was non-empty, so a list shorter than the bounds indexed out of range part-way through building the optimisation problem, naming neither argument and leaving `opt_prob` half-populated (S6466). The lengths are now compared up front and the mismatch is reported with both counts. --- src/hapi/calibration.py | 11 ++++++++++- src/hapi/catchment.py | 15 ++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/hapi/calibration.py b/src/hapi/calibration.py index 7a6eef2e..a9d11f2d 100644 --- a/src/hapi/calibration.py +++ b/src/hapi/calibration.py @@ -526,7 +526,8 @@ def lumpedCalibration( Raises: ValueError: If `basic_inputs` is missing required keys - `"Route"` or `"RoutingFn"`. + `"Route"` or `"RoutingFn"`, or if `"InitialValues"` is + given and does not hold one value per parameter. TypeError: If either bundle of optimization arguments is not a dict. """ @@ -593,6 +594,14 @@ def opt_fun(par): opt_prob = Optimization("HBV Calibration", opt_fun) if initial_values != []: + # One starting value per parameter. A shorter list used to index out of range + # part-way through building the problem, naming neither argument and leaving + # `opt_prob` half-populated. + if len(initial_values) != len(self.LB): + raise ValueError( + f"initial_values must hold one value per parameter; the bounds define " + f"{len(self.LB)} and {len(initial_values)} were given" + ) for i in range(len(self.LB)): opt_prob.addVar( f"x{i}", diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index 0fd5ead7..b38df18f 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -717,6 +717,8 @@ def read_lumped_model( None. Raises: + TypeError: If `initial_condition` is not a list, or if + `q_init` is given and is not a float. ValueError: If `lumped_model` is not a class or if `initial_condition` does not contain exactly 5 values. @@ -729,6 +731,14 @@ def read_lumped_model( self.lumped_model = lumped_model() self.area = catchment_area + # Typed before it is measured. The other order called `len` first, so None reported + # "object of type 'NoneType' has no len()" rather than naming the argument, and the + # `is not None` the type check then carried could never be false -- `len` would + # already have raised. + if not isinstance(initial_condition, list): + raise TypeError( + f"init_st should be of type list, got {type(initial_condition).__name__}" + ) if len(initial_condition) != 5: raise ValueError( f"state variables are 5 and the given initial values are {len(initial_condition)}" @@ -742,11 +752,6 @@ def read_lumped_model( ) self.q_init = q_init - if self.initial_cond is not None and not isinstance(self.initial_cond, list): - raise TypeError( - f"init_st should be of type list, got {type(self.initial_cond).__name__}" - ) - logger.debug("Lumped model is read successfully") def read_lumped_inputs(self, path: str): From fcac42e0743e2c3e348bfd814936e1772c8a50e5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:12:33 +0200 Subject: [PATCH 57/61] refactor(sonar): give each spatial resolution its own method Two functions were over the cognitive-complexity gate, and both for the same reason: a single body holding two branches that share nothing. `RunConfig`'s block validator was at 23 -- it grew this round with the inapplicable-field checks -- and `read_discharge_gauges` at 19. Each now dispatches on the resolution and nothing else, with the distributed and lumped bodies in methods of their own. A distributed run describes a grid, a routing network and a folder of per-gauge files; a lumped one describes two CSVs. Reading either no longer means skipping past the other, and the docstrings can say what each branch actually requires rather than covering both at once. No behaviour change: same checks, same messages, same order. --- src/hapi/catchment.py | 131 ++++++++++++++++++++++----------- src/hapi/config.py | 166 ++++++++++++++++++++++++------------------ 2 files changed, 182 insertions(+), 115 deletions(-) diff --git a/src/hapi/catchment.py b/src/hapi/catchment.py index b38df18f..7be98faa 100644 --- a/src/hapi/catchment.py +++ b/src/hapi/catchment.py @@ -979,51 +979,11 @@ def read_discharge_gauges( ind = pd.date_range(self.start, self.end, freq="h") if self.spatial_resolution.lower() == "distributed": - # `__init__` sets GaugesTable to None, so the `hasattr` this replaced was always - # true and never guarded anything: a caller who skipped `read_gauge_table` got a - # `TypeError` on None a few lines down instead. - if self.GaugesTable is None: - raise ValueError( - "the gauge table has not been read yet; call read_gauge_table before " - "read_discharge_gauges in distributed mode" - ) - - # The frame is labelled from `column` but every file is named after `id`, so the - # two are tracked separately: filling by `int(name)` instead of by the label the - # frame was built with left a `column != "id"` table with the requested columns - # all-NaN and a second set of id-named ones beside them, silently. - labels = self.GaugesTable[column].tolist() - self.QGauges = pd.DataFrame(index=ind, columns=labels) - - for i in range(len(self.GaugesTable)): - name = self.GaugesTable.loc[i, "id"] - if readfrom != "": - f = pd.read_csv( - f"{path}/{name}.csv", - index_col=0, - delimiter=delimiter, - skiprows=readfrom, - ) # ,#delimiter="\t" - else: - f = pd.read_csv( - f"{path}/{name}.csv", - header=0, - index_col=0, - delimiter=delimiter, - ) - - f.index = [dt.datetime.strptime(i, fmt) for i in f.index.tolist()] - self.QGauges[labels[i]] = f.loc[self.start : self.end, f.columns[-1]] + self._read_one_discharge_file_per_gauge( + path, ind, delimiter, column, fmt, readfrom + ) else: - if not os.path.exists(path): - raise FileNotFoundError( - f"The file you have entered{path} does not exist" - ) - - self.QGauges = pd.DataFrame(index=ind) - f = pd.read_csv(path, header=0, index_col=0, delimiter=delimiter) - f.index = [dt.datetime.strptime(i, fmt) for i in f.index.tolist()] - self.QGauges[f.columns[0]] = f.loc[self.start : self.end, f.columns[0]] + self._read_the_single_discharge_file(path, ind, delimiter, fmt) if split: if isinstance(start_date, str): @@ -1034,6 +994,89 @@ def read_discharge_gauges( logger.debug("Gauges data are read successfully") + def _read_one_discharge_file_per_gauge( + self, + path: str, + index: pd.DatetimeIndex, + delimiter: str, + column: str, + fmt: str, + readfrom: str, + ) -> None: + """Fill `QGauges` from a folder holding one CSV per gauge id. + + Args: + path: Folder of per-gauge CSVs, each named after a gauge id. + index: The model's date index, which the frame is built on. + delimiter: Discharge CSV delimiter. + column: Gauge-table column naming the frame's columns. + fmt: `strptime` format for each file's date column. + readfrom: Rows to skip, or "" to read from the header. + + Raises: + ValueError: `read_gauge_table` has not been called yet. + """ + # `__init__` sets GaugesTable to None, so the `hasattr` this replaced was always + # true and never guarded anything: a caller who skipped `read_gauge_table` got a + # `TypeError` on None a few lines down instead. + if self.GaugesTable is None: + raise ValueError( + "the gauge table has not been read yet; call read_gauge_table before " + "read_discharge_gauges in distributed mode" + ) + + # The frame is labelled from `column` but every file is named after `id`, so the + # two are tracked separately: filling by `int(name)` instead of by the label the + # frame was built with left a `column != "id"` table with the requested columns + # all-NaN and a second set of id-named ones beside them, silently. + labels = self.GaugesTable[column].tolist() + self.QGauges = pd.DataFrame(index=index, columns=labels) + + for i in range(len(self.GaugesTable)): + name = self.GaugesTable.loc[i, "id"] + if readfrom != "": + f = pd.read_csv( + f"{path}/{name}.csv", + index_col=0, + delimiter=delimiter, + skiprows=readfrom, + ) + else: + f = pd.read_csv( + f"{path}/{name}.csv", + header=0, + index_col=0, + delimiter=delimiter, + ) + + f.index = [dt.datetime.strptime(i, fmt) for i in f.index.tolist()] + self.QGauges[labels[i]] = f.loc[self.start : self.end, f.columns[-1]] + + def _read_the_single_discharge_file( + self, path: str, index: pd.DatetimeIndex, delimiter: str, fmt: str + ) -> None: + """Fill `QGauges` from one CSV, the lumped case. + + A lumped run has no grid to locate gauges on, so there is one hydrograph and no + gauge table: the frame takes the file's own first column as its only column. + + Args: + path: The discharge CSV. + index: The model's date index, which the frame is built on. + delimiter: Discharge CSV delimiter. + fmt: `strptime` format for the file's date column. + + Raises: + FileNotFoundError: `path` does not exist. + """ + if not os.path.exists(path): + raise FileNotFoundError(f"The file you have entered{path} does not exist") + + self.QGauges = pd.DataFrame(index=index) + f = pd.read_csv(path, header=0, index_col=0, delimiter=delimiter) + f.index = [dt.datetime.strptime(i, fmt) for i in f.index.tolist()] + self.QGauges[f.columns[0]] = f.loc[self.start : self.end, f.columns[0]] + def read_parameters_bound( self, upper_bound: list | np.ndarray, diff --git a/src/hapi/config.py b/src/hapi/config.py index 99cc58d5..0fea56dc 100644 --- a/src/hapi/config.py +++ b/src/hapi/config.py @@ -539,86 +539,110 @@ def _check_the_routing_method_matches_the_parameter_set(self) -> RunConfig: def _check_blocks_match_the_spatial_resolution(self) -> RunConfig: """Enforce the fields each spatial resolution requires. + The two resolutions share no rule -- one describes a grid and a routing network, the + other a pair of CSVs -- so each owns a method and this one only chooses between them. + Returns: RunConfig: This config, unchanged. Raises: - ValueError: A block the chosen `spatial_resolution` needs is missing, or one of the - three drivers a distributed `meteo.source` needs is unset. + ValueError: A block the chosen `spatial_resolution` needs is missing, or one it + will never read is present. """ if self.catchment.spatial_resolution == "distributed": - if self.flow_network is None: - raise ValueError( - "catchment.spatial_resolution is 'distributed', which needs a flow_network " - "block" - ) - # `flow_direction` is optional on the block because MAXBAS sends every cell straight - # to the outlet and never reads one. Muskingum routes along the network, so without - # it the build succeeds and `Run.RunHapi` dereferences a None array after every - # raster has been read. - if ( - self.catchment.routing_method == "muskingum" - and self.flow_network.flow_direction is None - ): - raise ValueError( - "catchment.routing_method is 'muskingum', which routes along the network, " - "so flow_network.flow_direction is required" - ) - # Only when gauges are configured at all: a distributed run that is not scored - # against observations omits the block entirely. - if self.gauges is not None and self.gauges.table is None: - raise ValueError( - "catchment.spatial_resolution is 'distributed', which needs gauges.table " - "to locate the gauges on the grid" - ) - missing = [ - name for name in METEO_DRIVERS if getattr(self.meteo, name) is None - ] - if missing: - raise ValueError(missing_drivers_message(missing)) - if self.meteo.source == "netcdf" and self.meteo.path is None: - raise ValueError(NETCDF_PATH_MESSAGE) - _reject_fields_the_run_will_not_read( - self.meteo, - _METEO_FIELDS_BY_SOURCE[self.meteo.source], - "meteo", - f"meteo.source is {self.meteo.source!r}", - ) + self._check_the_distributed_blocks() else: - if self.meteo.path is None: - raise ValueError( - "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the " - "CSV of catchment-average drivers" - ) - # `extra="forbid"` exists so a misspelled key fails rather than being dropped; - # accepting a correctly spelled but inapplicable block would be the same silence - # by another route. A lumped run has no grid, so neither block can be honoured. - if self.flow_network is not None: - raise ValueError( - "catchment.spatial_resolution is 'lumped', which has no grid, so a " - "flow_network block cannot be used" - ) - if self.meteo.source != "rasters": - raise ValueError( - f"catchment.spatial_resolution is 'lumped', which reads meteo.path as a " - f"CSV of catchment-average drivers; meteo.source " - f"{self.meteo.source!r} does not apply" - ) - lumped = "catchment.spatial_resolution is 'lumped'" + self._check_the_lumped_blocks() + return self + + def _check_the_distributed_blocks(self) -> None: + """Enforce what a distributed run needs and what its `meteo.source` can read. + + Raises: + ValueError: The routing network is missing or incomplete, the gauge table is + absent while gauges are configured, a driver is unset, `source="netcdf"` + names no file, or a field outside the source's own set is present. + """ + if self.flow_network is None: + raise ValueError( + "catchment.spatial_resolution is 'distributed', which needs a flow_network " + "block" + ) + # `flow_direction` is optional on the block because MAXBAS sends every cell straight + # to the outlet and never reads one. Muskingum routes along the network, so without + # it the build succeeds and `Run.RunHapi` dereferences a None array after every + # raster has been read. + if ( + self.catchment.routing_method == "muskingum" + and self.flow_network.flow_direction is None + ): + raise ValueError( + "catchment.routing_method is 'muskingum', which routes along the network, " + "so flow_network.flow_direction is required" + ) + # Only when gauges are configured at all: a distributed run that is not scored + # against observations omits the block entirely. + if self.gauges is not None and self.gauges.table is None: + raise ValueError( + "catchment.spatial_resolution is 'distributed', which needs gauges.table " + "to locate the gauges on the grid" + ) + + missing = [name for name in METEO_DRIVERS if getattr(self.meteo, name) is None] + if missing: + raise ValueError(missing_drivers_message(missing)) + if self.meteo.source == "netcdf" and self.meteo.path is None: + raise ValueError(NETCDF_PATH_MESSAGE) + + _reject_fields_the_run_will_not_read( + self.meteo, + _METEO_FIELDS_BY_SOURCE[self.meteo.source], + "meteo", + f"meteo.source is {self.meteo.source!r}", + ) + + def _check_the_lumped_blocks(self) -> None: + """Enforce what a lumped run needs, and refuse the grid it has no use for. + + `extra="forbid"` exists so a misspelled key fails rather than being dropped; + accepting a correctly spelled but inapplicable block would be the same silence by + another route. A lumped run has no grid, so nothing that describes one applies. + + Raises: + ValueError: `meteo.path` is unset, a routing network or a grid `meteo.source` is + present, or a `meteo` / `gauges` field this run will never read is set. + """ + if self.meteo.path is None: + raise ValueError( + "catchment.spatial_resolution is 'lumped', which needs meteo.path -- the " + "CSV of catchment-average drivers" + ) + if self.flow_network is not None: + raise ValueError( + "catchment.spatial_resolution is 'lumped', which has no grid, so a " + "flow_network block cannot be used" + ) + if self.meteo.source != "rasters": + raise ValueError( + f"catchment.spatial_resolution is 'lumped', which reads meteo.path as a " + f"CSV of catchment-average drivers; meteo.source " + f"{self.meteo.source!r} does not apply" + ) + + lumped = "catchment.spatial_resolution is 'lumped'" + _reject_fields_the_run_will_not_read( + self.meteo, + _LUMPED_METEO_FIELDS, + "meteo", + f"{lumped}, which reads meteo.path as one CSV of catchment-average drivers", + ) + if self.gauges is not None: _reject_fields_the_run_will_not_read( - self.meteo, - _LUMPED_METEO_FIELDS, - "meteo", - f"{lumped}, which reads meteo.path as one CSV of catchment-average drivers", + self.gauges, + _LUMPED_GAUGES_FIELDS, + "gauges", + f"{lumped}, which reads one discharge file and locates no gauges", ) - if self.gauges is not None: - _reject_fields_the_run_will_not_read( - self.gauges, - _LUMPED_GAUGES_FIELDS, - "gauges", - f"{lumped}, which reads one discharge file and locates no gauges", - ) - return self @model_validator(mode="after") def _check_the_dates_parse_and_are_ordered(self) -> RunConfig: From 2e1b909bccab1469e2e3b3773f6f56cf9fc55a2b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:16:28 +0200 Subject: [PATCH 58/61] test(sonar): leave one throwing call inside each pytest.raises block Eight `pytest.raises` blocks built their input inside the block -- `write_yaml` five times, a hand-built `MeteoConfig` twice, a `datetime` once -- so a failure in the setup would have been caught and read as the behaviour under test (S5778). Each is now built above the block, which also names it. Also: the assertion that the pre-flight check reports both missing paths is split in two, so a failure says which one was absent (S9073); the two fixtures drop the `scope="function"` that is already pytest's default (S9117); and the lumped example builds its scores dict as a literal (S7498). --- .../coello/run/coello-lumped-model-run.py | 2 +- tests/rrm/catchment/test_config.py | 51 ++++++++++++------- .../rrm/catchment/test_read_raster_inputs.py | 8 ++- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/examples/hydrological-model/coello/run/coello-lumped-model-run.py b/examples/hydrological-model/coello/run/coello-lumped-model-run.py index 83aa3fbb..16a9bf0c 100644 --- a/examples/hydrological-model/coello/run/coello-lumped-model-run.py +++ b/examples/hydrological-model/coello/run/coello-lumped-model-run.py @@ -31,7 +31,7 @@ Run.runLumped(Coello, Route, RoutingFn) # %% Calculate performance criteria -scores = dict() +scores = {} Qobs = Coello.QGauges["q"] diff --git a/tests/rrm/catchment/test_config.py b/tests/rrm/catchment/test_config.py index b6a4305d..ba8beb20 100644 --- a/tests/rrm/catchment/test_config.py +++ b/tests/rrm/catchment/test_config.py @@ -41,7 +41,7 @@ COMBINED_NC = "tests/rrm/data/coello/meteo.nc" -@pytest.fixture(scope="function") +@pytest.fixture def distributed_mapping( coello_start_date: str, coello_end_date: str, @@ -86,7 +86,7 @@ def distributed_mapping( } -@pytest.fixture(scope="function") +@pytest.fixture def lumped_mapping( coello_start_date: str, coello_end_date: str, @@ -1132,8 +1132,10 @@ def test_a_config_missing_a_driver_is_refused(self): built directly -- which is exactly when the message naming the missing drivers is the only thing the caller has to go on. """ + config = MeteoConfig(source="rasters", precipitation="p") + with pytest.raises(ValueError, match="all three meteorological drivers") as exc: - MeteoInputs.from_config(MeteoConfig(source="rasters", precipitation="p")) + MeteoInputs.from_config(config) assert "temperature" in str(exc.value), ( f"the error should name what is unset: {exc.value}" @@ -1146,15 +1148,15 @@ def test_the_netcdf_source_needs_a_path(self): For this source the driver fields are variable names inside one file, so without the file there is nothing to read them from. """ + config = MeteoConfig( + source="netcdf", + precipitation="precipitation", + temperature="temperature", + evapotranspiration="evapotranspiration", + ) + with pytest.raises(ValueError, match="meteo.path must be set"): - MeteoInputs.from_config( - MeteoConfig( - source="netcdf", - precipitation="precipitation", - temperature="temperature", - evapotranspiration="evapotranspiration", - ) - ) + MeteoInputs.from_config(config) def test_the_netcdf_source_reads_the_three_variables_from_one_file( self, coello_start_date: str, coello_end_date: str @@ -1451,8 +1453,10 @@ def test_an_unregistered_model_class_is_refused_by_name( """ distributed_mapping["conceptual_model"]["model_class"] = "HBV97" + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(ValueError, match="not.*registered") as exc: - Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Catchment.from_yaml(path) assert "HBVBergestrom92" in str(exc.value), ( f"the error should list the known models: {exc.value}" @@ -1479,8 +1483,10 @@ def test_an_unregistered_model_class_is_refused_before_the_readers_run( distributed_mapping["flow_network"]["flow_accumulation"] = "no/such/acc.tif" distributed_mapping["flow_network"]["flow_direction"] = "no/such/fd.tif" + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(ValueError, match="not.*registered"): - Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Catchment.from_yaml(path) @pytest.mark.parametrize("cls", [Catchment, Calibration]) def test_the_builder_returns_the_class_it_was_called_on( @@ -1800,8 +1806,10 @@ def test_an_invalid_configuration_fails_before_anything_is_read( """ distributed_mapping["catchment"]["spatial_resolution"] = "semi" + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(ValidationError, match="Input should be"): - Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Catchment.from_yaml(path) def test_the_configuration_is_read_from_the_given_path( self, distributed_mapping, tmp_path @@ -1892,12 +1900,17 @@ def test_every_missing_input_path_is_named_at_once( distributed_mapping["parameters"]["path"] = "no/such/parameters" distributed_mapping["gauges"]["table"] = "no/such/gauges.csv" + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(FileNotFoundError) as exc: - Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Catchment.from_yaml(path) message = str(exc.value) - assert "parameters.path" in message and "gauges.table" in message, ( - f"both missing paths should be named in one message: {message}" + assert "parameters.path" in message, ( + f"the missing parameter folder should be named: {message}" + ) + assert "gauges.table" in message, ( + f"the missing gauge table should be named in the same message: {message}" ) def test_a_netcdf_variable_name_is_not_checked_for_existence( @@ -2066,5 +2079,7 @@ def test_a_missing_driver_folder_is_reported_before_anything_is_read( "evapotranspiration": str(Path(coello_evap_path).resolve()), } + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(FileNotFoundError, match="meteo.precipitation"): - Catchment.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Catchment.from_yaml(path) diff --git a/tests/rrm/catchment/test_read_raster_inputs.py b/tests/rrm/catchment/test_read_raster_inputs.py index 914d1cfa..b5310240 100644 --- a/tests/rrm/catchment/test_read_raster_inputs.py +++ b/tests/rrm/catchment/test_read_raster_inputs.py @@ -605,12 +605,10 @@ def test_a_datetime_bound_needs_the_date_ordering(self, coello_prec_path, bound) bounds are indices and a datetime has no meaning -- `int()` would fail on it several frames down, in a message naming neither the argument nor the mode. """ + bounds = {bound: dt.datetime(2009, 1, 1)} + with pytest.raises(TypeError, match="needs date=True") as exc: - read_rasters( - coello_prec_path, - date=False, - **{bound: dt.datetime(2009, 1, 1)}, - ) + read_rasters(coello_prec_path, date=False, **bounds) assert "indices" in str(exc.value), ( f"the error should say what the bounds mean in this mode: {exc.value}" From f189ed0ae987b76a45b361263c5826b5416c3a5f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:22:52 +0200 Subject: [PATCH 59/61] refactor(sonar): declare the optimisation variables in one place Three calibration entry points each wrote their own loop adding one bounded variable per parameter, and the lumped one wrote it twice -- once seeded with a starting point and once not. Folding the length check into that duplicated pair is what pushed `lumpedCalibration` over the cognitive-complexity gate (S3776). One helper now declares the variables for all three, taking the optional starting point, so the bounds-length check lives beside the loop it protects rather than beside one of four copies. Also hoists the last `write_yaml` out of a `pytest.raises` block (S5778). --- src/hapi/calibration.py | 59 ++++++++++++++++++------------ tests/rrm/catchment/test_config.py | 4 +- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/hapi/calibration.py b/src/hapi/calibration.py index a9d11f2d..8324d755 100644 --- a/src/hapi/calibration.py +++ b/src/hapi/calibration.py @@ -102,6 +102,38 @@ def __init__( self.OFArgs: list | None = None self.OFvalue: float | None = None + def _declare_the_parameter_variables( + self, opt_prob: Optimization, initial_values: list | None = None + ) -> None: + """Add one continuous optimisation variable per parameter, bounded by LB and UB. + + Every calibration entry point declares the same variables the same way; only the + lumped one can also seed them with a starting point. + + Args: + opt_prob: The problem being built. + initial_values: One starting value per parameter, or None to let the optimiser + choose. + + Raises: + ValueError: `initial_values` is given and does not hold one value per parameter. + """ + # One starting value per parameter. A shorter list used to index out of range + # part-way through building the problem, naming neither argument and leaving + # `opt_prob` half-populated. + seeded = initial_values is not None and len(initial_values) > 0 + if seeded and len(initial_values) != len(self.LB): + raise ValueError( + f"initial_values must hold one value per parameter; the bounds define " + f"{len(self.LB)} and {len(initial_values)} were given" + ) + + for i in range(len(self.LB)): + seed = {"value": initial_values[i]} if seeded else {} + opt_prob.addVar( + f"x{i}", type="c", lower=self.LB[i], upper=self.UB[i], **seed + ) + def read_objective_function( self, objective_function: Callable[..., Any], args: list | None ): @@ -315,8 +347,7 @@ def opt_fun(par): ### define the optimization components opt_prob = Optimization("HBV Calibration", opt_fun) - for i in range(len(self.LB)): - opt_prob.addVar(f"x{i}", type="c", lower=self.LB[i], upper=self.UB[i]) + self._declare_the_parameter_variables(opt_prob) opt_prob.addObj("f") @@ -453,8 +484,7 @@ def opt_fun(par): # define the optimization components opt_prob = Optimization("HBV Calibration", opt_fun) - for i in range(len(self.LB)): - opt_prob.addVar(f"x{i}", type="c", lower=self.LB[i], upper=self.UB[i]) + self._declare_the_parameter_variables(opt_prob) print(opt_prob) @@ -593,26 +623,7 @@ def opt_fun(par): ### define the optimization components opt_prob = Optimization("HBV Calibration", opt_fun) - if initial_values != []: - # One starting value per parameter. A shorter list used to index out of range - # part-way through building the problem, naming neither argument and leaving - # `opt_prob` half-populated. - if len(initial_values) != len(self.LB): - raise ValueError( - f"initial_values must hold one value per parameter; the bounds define " - f"{len(self.LB)} and {len(initial_values)} were given" - ) - for i in range(len(self.LB)): - opt_prob.addVar( - f"x{i}", - type="c", - lower=self.LB[i], - upper=self.UB[i], - value=initial_values[i], - ) - else: - for i in range(len(self.LB)): - opt_prob.addVar(f"x{i}", type="c", lower=self.LB[i], upper=self.UB[i]) + self._declare_the_parameter_variables(opt_prob, initial_values) opt_prob.addObj("f") diff --git a/tests/rrm/catchment/test_config.py b/tests/rrm/catchment/test_config.py index ba8beb20..1757815d 100644 --- a/tests/rrm/catchment/test_config.py +++ b/tests/rrm/catchment/test_config.py @@ -1526,8 +1526,10 @@ def test_run_cannot_be_built_because_it_takes_no_constructor_arguments( constructor arity produce a `TypeError` about an unexpected keyword argument -- an error that says nothing about what to do instead. """ + path = write_yaml(distributed_mapping, tmp_path) + with pytest.raises(TypeError, match="Catchment.from_yaml"): - Run.from_yaml(write_yaml(distributed_mapping, tmp_path)) + Run.from_yaml(path) def test_a_lumped_configuration_reads_the_averaged_driver_csv( self, lumped_mapping, tmp_path From 6251b4e5e84020bc41073edeaefe3c56dc020516 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:45:46 +0200 Subject: [PATCH 60/61] test(calibration): cover the three guards this session's Sonar fixes added The pre-merge check's diff-coverage pass found three untested branches, two of them new this session: - `lumpedCalibration`'s `initial_values`-length mismatch guard (the S6466 IndexError fix) -- nothing called it with a mismatched list. - `read_lumped_model`'s `not isinstance(initial_condition, list)` guard (the S2589 always-true-condition fix) -- nothing called it with a non-list. - `_check_optimization_args`'s two dict-type checks -- pre-existing on this branch, never exercised by any test. Each is now pinned: the mismatch test confirms the optimiser is never reached and the error names both lengths; the non-list test parametrises over a tuple, an array and a string that all happen to hold five elements, so the length-first ordering the old bug depended on cannot silently come back; the dict-type tests cover both argument bundles and confirm the optimiser is unreached. --- .../test_calibration_distributed.py | 87 +++++++++++++++++++ .../test_read_parameters_validation.py | 53 +++++++++++ 2 files changed, 140 insertions(+) diff --git a/tests/rrm/calibration/test_calibration_distributed.py b/tests/rrm/calibration/test_calibration_distributed.py index b23eb664..841aad48 100644 --- a/tests/rrm/calibration/test_calibration_distributed.py +++ b/tests/rrm/calibration/test_calibration_distributed.py @@ -333,6 +333,54 @@ def test_stores_the_optimizer_result_on_the_instance( ) +class TestCheckOptimizationArgs: + """Tests for `_check_optimization_args`, the guard every entry point runs first.""" + + @pytest.mark.parametrize( + "args, bad_index, bad_kind", + [ + (["not-a-dict", None, {}], 0, "objective-function"), + ([{}, None, "not-a-dict"], 2, "solver"), + ], + ids=["objective-args", "solver-args"], + ) + def test_a_non_dict_bundle_is_refused_before_the_optimizer_is_built( + self, + gauged_calibration: Calibration, + stub_optimizer: dict, + spatial_var_stub, + args, + bad_index, + bad_kind, + ): + """Test that each of the two argument bundles is checked, naming which one. + + Args: + gauged_calibration: A ready-to-run distributed Calibration. + stub_optimizer: Records whether the optimiser was reached. + spatial_var_stub: Stand-in for the spatial parameter function. + args: The `[api_obj_args, pll_type, api_solve_args]` triple, one entry bad. + bad_index: Which position in `args` carries the non-dict value. + bad_kind: The word the error message should use for that position. + + Test scenario: + Both bundles are unpacked with `**` inside Oasis, so anything but a dict fails + there instead of at the call that supplied it -- unless this guard catches it + first, before `Optimization(...)` and the harmony-search engine are built at all. + """ + coello = gauged_calibration + coello.read_objective_function(metrics.rmse, []) + coello.LB = np.zeros(12) + coello.UB = np.ones(12) + + with pytest.raises(TypeError, match=f"{bad_kind} arguments should be a dict"): + coello.run_calibration(spatial_var_stub, args) + + assert "solve_kwargs" not in stub_optimizer, ( + "the optimiser must not be reached when an argument bundle is malformed" + ) + + class TestLumpedCalibration: """Tests for `Calibration.lumpedCalibration`.""" @@ -398,6 +446,45 @@ def test_initial_values_are_seeded_into_the_problem( f"Expected one variable per bound (12), got {stub_optimizer['n_vars']}" ) + def test_a_mismatched_initial_values_length_is_refused( + self, + coello_rrm_date: list, + lumped_meteo_data_path: str, + stub_optimizer: dict, + ): + """Test that `InitialValues` shorter than the bounds is rejected, not indexed out of range. + + Args: + coello_rrm_date: [start, end] dates for the lumped fixture. + lumped_meteo_data_path: CSV of catchment-average drivers. + stub_optimizer: Records whether the optimiser was reached. + + Test scenario: + The seeded branch loops `range(len(self.LB))` and indexes `initial_values[i]`, so + a shorter list used to run past its end partway through building the problem, + leaving `opt_prob` half-populated and raising `IndexError` far from the call that + supplied the list. The length is now compared up front. + """ + coello = Calibration("rrm", coello_rrm_date[0], coello_rrm_date[1]) + coello.read_lumped_inputs(lumped_meteo_data_path) + coello.LB = np.zeros(12) + coello.UB = np.ones(12) + basic_inputs = dict( + Route=0, + RoutingFn=Routing.triangular_routing_1, + InitialValues=[0.5, 0.5, 0.5], + ) + + with pytest.raises(ValueError, match="one value per parameter") as exc: + coello.lumpedCalibration(basic_inputs, _optimization_args()) + + assert "3" in str(exc.value) and "12" in str(exc.value), ( + f"the error should name both lengths: {exc.value}" + ) + assert "n_vars" not in stub_optimizer, ( + "the optimiser must not be reached when the seed does not match the bounds" + ) + def _pairwise_objective(qgauges, gauges_table) -> float: """Score a trial the way `run_calibration` actually calls the objective. diff --git a/tests/rrm/catchment/test_read_parameters_validation.py b/tests/rrm/catchment/test_read_parameters_validation.py index 79929326..b06d01bb 100644 --- a/tests/rrm/catchment/test_read_parameters_validation.py +++ b/tests/rrm/catchment/test_read_parameters_validation.py @@ -330,6 +330,59 @@ def test_a_non_float_initial_discharge_is_refused( model.read_lumped_model(HBVLumped, 1530.0, coello_initial_cond, q_init=bad) +class TestReadLumpedModelInitialCondition: + """Tests for the `initial_condition` type guard in `Catchment.read_lumped_model`.""" + + def test_a_list_of_five_is_accepted( + self, coello_start_date: str, coello_end_date: str, coello_initial_cond: list + ): + """Test that the documented type and length pass through unchanged. + + Test scenario: + The happy path the guard must not disturb: a five-element list is stored as + `initial_cond` verbatim. + """ + model = Catchment("coello", coello_start_date, coello_end_date) + + model.read_lumped_model(HBVLumped, 1530.0, coello_initial_cond) + + assert model.initial_cond == coello_initial_cond, ( + f"the initial condition must be stored unchanged, got {model.initial_cond}" + ) + + @pytest.mark.parametrize( + "bad", + [(0, 10, 10, 10, 0), np.zeros(5), "01010", None], + ids=["tuple", "array", "str", "none"], + ) + def test_a_non_list_initial_condition_is_refused( + self, coello_start_date: str, coello_end_date: str, bad + ): + """Test that anything other than a list is rejected before its length is measured. + + Args: + coello_start_date: Simulation start date. + coello_end_date: Simulation end date. + bad: A value of the wrong type, including one with the right effective length. + + Test scenario: + The check used to measure `len(initial_condition)` before typing it, so a value + `len()` cannot take -- `None` -- reported "object of type 'NoneType' has no + len()" rather than naming the argument, and the `is not None` a later type check + carried could never be false, because `len` would already have raised. Typing + first means every non-list is refused the same way, including a tuple or array + that happens to hold five elements. + """ + model = Catchment("coello", coello_start_date, coello_end_date) + + with pytest.raises(TypeError, match="init_st should be of type list") as exc: + model.read_lumped_model(HBVLumped, 1530.0, bad) + + assert type(bad).__name__ in str(exc.value), ( + f"the error should name what it got: {exc.value}" + ) + + class TestReadLumpedInputs: """Tests for the column handling in `Catchment.read_lumped_inputs`.""" From 676682efb709320c81e73b7924971fe7b85698c1 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 31 Aug 2026 00:53:19 +0200 Subject: [PATCH 61/61] test(sonar): split the new guard test's raises block and assertion The mismatch-length test added in the previous commit built `_optimization_args()` inside the `pytest.raises` block alongside the call under test (S5778), and asserted both halves of the error message with `and` (S9073) -- the two patterns every other test in this file had already been fixed to avoid. --- tests/rrm/calibration/test_calibration_distributed.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/rrm/calibration/test_calibration_distributed.py b/tests/rrm/calibration/test_calibration_distributed.py index 841aad48..c4515351 100644 --- a/tests/rrm/calibration/test_calibration_distributed.py +++ b/tests/rrm/calibration/test_calibration_distributed.py @@ -475,11 +475,16 @@ def test_a_mismatched_initial_values_length_is_refused( InitialValues=[0.5, 0.5, 0.5], ) + optimization_args = _optimization_args() + with pytest.raises(ValueError, match="one value per parameter") as exc: - coello.lumpedCalibration(basic_inputs, _optimization_args()) + coello.lumpedCalibration(basic_inputs, optimization_args) - assert "3" in str(exc.value) and "12" in str(exc.value), ( - f"the error should name both lengths: {exc.value}" + assert "3" in str(exc.value), ( + f"the error should name the given length: {exc.value}" + ) + assert "12" in str(exc.value), ( + f"the error should name the expected length too: {exc.value}" ) assert "n_vars" not in stub_optimizer, ( "the optimiser must not be reached when the seed does not match the bounds"