From 13914b21eda3f1fa7e10cf1fa7f12c4060e7eddc Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 28 Jul 2026 16:58:53 -0600 Subject: [PATCH] Add optional pint unit conversion and per-client fixture tests --- metloom/pointdata/base.py | 74 ++- metloom/pointdata/cdec.py | 27 +- metloom/pointdata/cues.py | 15 +- metloom/pointdata/files.py | 15 +- metloom/pointdata/geosphere_austria.py | 19 +- metloom/pointdata/mesowest.py | 18 +- metloom/pointdata/norway.py | 16 +- metloom/pointdata/nws_forecast.py | 17 +- metloom/pointdata/sail.py | 14 +- metloom/pointdata/snotel.py | 21 +- metloom/pointdata/usgs.py | 19 +- metloom/unit_conversions.py | 133 ++++ setup.py | 1 + tests/capture_unit_fixtures.py | 232 +++++++ tests/data/unit_fixtures/cdec.pkl | Bin 0 -> 941 bytes tests/data/unit_fixtures/csas_sbsp.pkl | Bin 0 -> 13508 bytes tests/data/unit_fixtures/cues_daily.pkl | Bin 0 -> 213 bytes tests/data/unit_fixtures/geosphere_hist.pkl | Bin 0 -> 515 bytes tests/data/unit_fixtures/mesowest.pkl | Bin 0 -> 347 bytes tests/data/unit_fixtures/norway_hourly.pkl | Bin 0 -> 16704 bytes tests/data/unit_fixtures/nws.pkl | Bin 0 -> 65997 bytes tests/data/unit_fixtures/sail_precip.pkl | Bin 0 -> 1082 bytes tests/data/unit_fixtures/snotel_daily.pkl | Bin 0 -> 324 bytes tests/data/unit_fixtures/snowex_lsos.pkl | Bin 0 -> 7566 bytes tests/data/unit_fixtures/usgs_daily.pkl | Bin 0 -> 2568 bytes tests/test_unit_conversion.py | 636 ++++++++++++++++++++ 26 files changed, 1217 insertions(+), 40 deletions(-) create mode 100644 metloom/unit_conversions.py create mode 100644 tests/capture_unit_fixtures.py create mode 100644 tests/data/unit_fixtures/cdec.pkl create mode 100644 tests/data/unit_fixtures/csas_sbsp.pkl create mode 100644 tests/data/unit_fixtures/cues_daily.pkl create mode 100644 tests/data/unit_fixtures/geosphere_hist.pkl create mode 100644 tests/data/unit_fixtures/mesowest.pkl create mode 100644 tests/data/unit_fixtures/norway_hourly.pkl create mode 100644 tests/data/unit_fixtures/nws.pkl create mode 100644 tests/data/unit_fixtures/sail_precip.pkl create mode 100644 tests/data/unit_fixtures/snotel_daily.pkl create mode 100644 tests/data/unit_fixtures/snowex_lsos.pkl create mode 100644 tests/data/unit_fixtures/usgs_daily.pkl create mode 100644 tests/test_unit_conversion.py diff --git a/metloom/pointdata/base.py b/metloom/pointdata/base.py index 47f0591..eefbc78 100644 --- a/metloom/pointdata/base.py +++ b/metloom/pointdata/base.py @@ -1,12 +1,13 @@ import copy import logging from datetime import datetime -from typing import List +from typing import Dict, List, Union import pandas as pd import geopandas as gpd +from ..unit_conversions import convert_series from ..variables import SensorDescription, VariableBase LOG = logging.getLogger("metloom.pointdata.base") @@ -168,6 +169,62 @@ def validate_sensor_df(cls, gdf: gpd.GeoDataFrame): if "_units" not in rc: assert f"{rc}_units" in remaining_columns + def _convert_units( + self, + gdf: gpd.GeoDataFrame, + desired_units: Union[str, Dict[str, str], None], + ): + """ + Optionally convert variable columns to a desired unit using pint. + + This is a no-op unless ``desired_units`` is provided - when it is + ``None`` pint is never invoked and the dataframe is returned unchanged. + + Args: + gdf: the assembled GeoDataFrame returned by a ``get_*`` method. + May be ``None`` or empty. + desired_units: either a single unit string applied to every + variable column, or a dict mapping ``variable.name`` to a unit + string. Units strings are pint-compatible (e.g. ``"degC"``, + ``"mm"``, ``"W/m^2"``). + Returns: + The GeoDataFrame with the requested variable columns converted in + place. Converted values are plain magnitudes (never pint + Quantities) and the matching ``{name}_units`` column is updated to + the target unit. NaNs are preserved. + """ + # Short-circuit before touching pint when no conversion was requested + if desired_units is None or gdf is None or len(gdf) == 0: + return gdf + + # variable columns are those that have a sibling `{col}_units` column + variable_columns = [ + c for c in gdf.columns if f"{c}_units" in gdf.columns + ] + for column in variable_columns: + if isinstance(desired_units, str): + target = desired_units + else: + target = desired_units.get(column) + if target is None: + continue + + units_column = f"{column}_units" + # infer the current unit from the first non-null entry + current = gdf[units_column].dropna() + if len(current) == 0: + LOG.warning( + "No units found for %s; skipping conversion", column + ) + continue + source_unit = current.iloc[0] + + gdf[column] = convert_series( + gdf[column].to_numpy(), source_unit, target + ) + gdf[units_column] = target + return gdf + def __repr__(self): return f"{self.__class__.__name__}({self.id!r}, {self.name!r})" @@ -197,6 +254,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get daily measurement data @@ -205,6 +263,10 @@ def get_daily_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns @@ -222,6 +284,7 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get hourly measurement data @@ -230,6 +293,10 @@ def get_hourly_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns @@ -247,6 +314,7 @@ def get_snow_course_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get snow course data @@ -255,6 +323,10 @@ def get_snow_course_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns diff --git a/metloom/pointdata/cdec.py b/metloom/pointdata/cdec.py index 838e301..6ae5222 100644 --- a/metloom/pointdata/cdec.py +++ b/metloom/pointdata/cdec.py @@ -228,7 +228,8 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], duration_list: List[str], - include_measurement_date=False + include_measurement_date=False, + desired_units=None, ): """ Args: @@ -240,6 +241,8 @@ def _get_data( include_measurement_date: boolean for including the 'measurmentDate' column in the resulting dataframe. This column is only relevant for snow courses + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -280,6 +283,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_event_data( @@ -287,14 +291,18 @@ def get_event_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): - return self._get_data(start_date, end_date, variables, ["E"]) + return self._get_data( + start_date, end_date, variables, ["E"], desired_units=desired_units + ) def get_daily_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -302,24 +310,32 @@ def get_daily_data( https://cdec.water.ca.gov/dynamicapp/req/JSONDataServlet? Stations=TNY&SensorNums=3&dur_code=D&Start=2021-05-16&End=2021-05-16 """ - return self._get_data(start_date, end_date, variables, ["D", "h", "E"]) + return self._get_data( + start_date, end_date, variables, ["D", "h", "E"], + desired_units=desired_units, + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ - return self._get_data(start_date, end_date, variables, ["h", "E"]) + return self._get_data( + start_date, end_date, variables, ["h", "E"], + desired_units=desired_units, + ) def get_snow_course_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_snow_course_data @@ -327,7 +343,8 @@ def get_snow_course_data( if not self.is_partly_snow_course(): raise ValueError(f"{self.id} is not a snow course") return self._get_data( - start_date, end_date, variables, ["M"], include_measurement_date=True + start_date, end_date, variables, ["M"], + include_measurement_date=True, desired_units=desired_units, ) @staticmethod diff --git a/metloom/pointdata/cues.py b/metloom/pointdata/cues.py index bf89cdb..9c7b76a 100644 --- a/metloom/pointdata/cues.py +++ b/metloom/pointdata/cues.py @@ -114,7 +114,7 @@ def _sensor_response_to_df(self, data, variable): def _get_data( self, start_date, end_date, variables: List[SensorDescription], - period, + period, desired_units=None, ): df = pd.DataFrame() df.index.name = "datetime" @@ -133,18 +133,23 @@ def _get_data( df = gpd.GeoDataFrame(df, geometry=[self.metadata] * len(df)) df = df.reset_index().set_index(["datetime", "site"]) self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "day" + start_date, end_date, variables, "day", + desired_units=desired_units, ) def get_hourly_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "hr" + start_date, end_date, variables, "hr", + desired_units=desired_units, ) def get_snow_course_data(self, start_date: datetime, end_date: datetime, diff --git a/metloom/pointdata/files.py b/metloom/pointdata/files.py index 7e445b3..54887b9 100644 --- a/metloom/pointdata/files.py +++ b/metloom/pointdata/files.py @@ -169,7 +169,7 @@ def _get_one_variable(self, resp_df, period, variable: SensorDescription): return data def _get_data(self, start_date, end_date, variables: List[SensorDescription], - period): + period, desired_units=None): """ Utilizes cached data or downloads the data """ @@ -222,18 +222,23 @@ def _get_data(self, start_date, end_date, variables: List[SensorDescription], df = df.reset_index().set_index(["datetime", "site"]) self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "D" + start_date, end_date, variables, "D", + desired_units=desired_units, ) def get_hourly_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "h" + start_date, end_date, variables, "h", + desired_units=desired_units, ) def _get_metadata(self): diff --git a/metloom/pointdata/geosphere_austria.py b/metloom/pointdata/geosphere_austria.py index cdaa754..4dd7f0f 100644 --- a/metloom/pointdata/geosphere_austria.py +++ b/metloom/pointdata/geosphere_austria.py @@ -172,6 +172,7 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], desired_duration: str, + desired_units=None, ): """ @@ -181,6 +182,8 @@ def _get_data( variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES desired_duration: duration code ['D', 'h'] + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -212,6 +215,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df @classmethod @@ -346,6 +350,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -354,19 +359,24 @@ def get_daily_data( tawes-v1-10min?parameters=TL&station_ids=11035 """ self._validate_dates(end_date) - return self._get_data(start_date, end_date, variables, "D") + return self._get_data( + start_date, end_date, variables, "D", desired_units=desired_units + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ self._validate_dates(end_date) - return self._get_data(start_date, end_date, variables, "h") + return self._get_data( + start_date, end_date, variables, "h", desired_units=desired_units + ) class GeoSphereHistPointData(GeoSpherePointDataBase): @@ -406,6 +416,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -413,4 +424,6 @@ def get_daily_data( https://dataset.api.hub.geosphere.at/v1/station/historical/klima-v1-1d ?station_ids=11401&start=2023-04-12&end=2023-04-14¶meters=schnee """ - return self._get_data(start_date, end_date, variables, None) + return self._get_data( + start_date, end_date, variables, None, desired_units=desired_units + ) diff --git a/metloom/pointdata/mesowest.py b/metloom/pointdata/mesowest.py index cd51427..bea0244 100644 --- a/metloom/pointdata/mesowest.py +++ b/metloom/pointdata/mesowest.py @@ -91,7 +91,8 @@ def _get_data(self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], - interval='H'): + interval='H', + desired_units=None): """ Make get request to Mesowest and return JSON Args: @@ -100,6 +101,8 @@ def _get_data(self, variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES interval: String interval the resulting data is resampled to + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: dictionary of response values """ @@ -136,6 +139,7 @@ def _get_data(self, else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df @staticmethod @@ -244,6 +248,7 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get hourly measurement data @@ -262,7 +267,10 @@ def get_hourly_data( TestCDECStation.tny_daily_expected for example dataframe. Datetimes should be in UTC """ - df = self._get_data(start_date, end_date, variables, interval='H') + df = self._get_data( + start_date, end_date, variables, interval='H', + desired_units=desired_units, + ) return df def get_daily_data( @@ -270,6 +278,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get daily measurement data @@ -288,7 +297,10 @@ def get_daily_data( TestCDECStation.tny_daily_expected for example dataframe. Datetimes should be in UTC """ - df = self._get_data(start_date, end_date, variables, interval='D') + df = self._get_data( + start_date, end_date, variables, interval='D', + desired_units=desired_units, + ) return df @classmethod diff --git a/metloom/pointdata/norway.py b/metloom/pointdata/norway.py index bdbbe7c..3d9c1dc 100644 --- a/metloom/pointdata/norway.py +++ b/metloom/pointdata/norway.py @@ -360,6 +360,7 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], desired_duration=None, + desired_units=None, ): """ Args: @@ -367,6 +368,8 @@ def _get_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -396,6 +399,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data( @@ -403,12 +407,14 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data """ return self._get_data( - start_date, end_date, variables, desired_duration="D" + start_date, end_date, variables, desired_duration="D", + desired_units=desired_units, ) def get_hourly_data( @@ -416,12 +422,14 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ return self._get_data( - start_date, end_date, variables, desired_duration="h" + start_date, end_date, variables, desired_duration="h", + desired_units=desired_units, ) def get_event_data( @@ -429,12 +437,14 @@ def get_event_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get data in original frequency from API """ return self._get_data( - start_date, end_date, variables, desired_duration=None + start_date, end_date, variables, desired_duration=None, + desired_units=desired_units, ) @classmethod diff --git a/metloom/pointdata/nws_forecast.py b/metloom/pointdata/nws_forecast.py index 5ebb64b..0d8e479 100644 --- a/metloom/pointdata/nws_forecast.py +++ b/metloom/pointdata/nws_forecast.py @@ -212,6 +212,7 @@ def _get_data( self, variables: List[SensorDescription], desired_duration=None, + desired_units=None, ): """ Args: @@ -219,6 +220,8 @@ def _get_data( from self.ALLOWED_VARIABLES desired_duration: desired resample duration ("D", "h"). Data is hourly be default + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -247,11 +250,13 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with daily results for a 7 day forecast. @@ -260,11 +265,14 @@ def get_daily_forecast( Args: variables: list of variables to return """ - return self._get_data(variables, desired_duration="D") + return self._get_data( + variables, desired_duration="D", desired_units=desired_units + ) def get_hourly_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with hourly results for a 7 day forecast. @@ -273,11 +281,14 @@ def get_hourly_forecast( Args: variables: list of variables to return """ - return self._get_data(variables, desired_duration="h") + return self._get_data( + variables, desired_duration="h", desired_units=desired_units + ) def get_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with hourly results for a 7 day forecast. @@ -287,4 +298,4 @@ def get_forecast( variables: list of variables to return """ # Do not resample - return self._get_data(variables) + return self._get_data(variables, desired_units=desired_units) diff --git a/metloom/pointdata/sail.py b/metloom/pointdata/sail.py index cac219c..06ff9e4 100644 --- a/metloom/pointdata/sail.py +++ b/metloom/pointdata/sail.py @@ -52,18 +52,26 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: list[SensorDescription], + desired_units=None, ): self._check_start_end_dates(start_date, end_date) - return self._download_sail_data(start_date, end_date, variables, interval="D") + return self._download_sail_data( + start_date, end_date, variables, interval="D", + desired_units=desired_units, + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: list[SensorDescription], + desired_units=None, ): self._check_start_end_dates(start_date, end_date) - return self._download_sail_data(start_date, end_date, variables, interval="h") + return self._download_sail_data( + start_date, end_date, variables, interval="h", + desired_units=desired_units, + ) def _download_sail_data( self, @@ -71,6 +79,7 @@ def _download_sail_data( end_date: datetime, variables: list[SensorDescription], interval: str, + desired_units=None, ) -> pd.DataFrame: """ The ARM data is stored in a series of files based on the sensors at the location. @@ -121,6 +130,7 @@ def _download_sail_data( df["datasource"] = "ARM" df.reset_index(inplace=True) df = df.set_index(["datetime", "site"]) + df = self._convert_units(df, desired_units) return df else: LOG.error( diff --git a/metloom/pointdata/snotel.py b/metloom/pointdata/snotel.py index f978d92..b0709c6 100644 --- a/metloom/pointdata/snotel.py +++ b/metloom/pointdata/snotel.py @@ -112,6 +112,7 @@ def _fetch_data_for_variables(self, client: SeriesSnotelClient, variables: List[SensorDescription], duration: str, include_measurement_date=False, + desired_units=None, ): result_map = {} for variable in variables: @@ -121,15 +122,17 @@ def _fetch_data_for_variables(self, client: SeriesSnotelClient, result_map[variable] = data else: LOG.warning(f"No {variable.name} found for {self.name}") - return self._snotel_response_to_df( + df = self._snotel_response_to_df( result_map, duration, include_measurement_date=include_measurement_date ) + return self._convert_units(df, desired_units) def get_daily_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -139,14 +142,16 @@ def get_daily_data( begin_date=start_date, end_date=end_date, ) - return self._fetch_data_for_variables(client, variables, - client.DURATION) + return self._fetch_data_for_variables( + client, variables, client.DURATION, desired_units=desired_units + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription] + variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data @@ -157,13 +162,16 @@ def get_hourly_data( begin_date=start_date, end_date=end_date, ) - return self._fetch_data_for_variables(client, variables, "HOURLY") + return self._fetch_data_for_variables( + client, variables, "HOURLY", desired_units=desired_units + ) def get_snow_course_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_snow_course_data @@ -174,7 +182,8 @@ def get_snow_course_data( end_date=end_date, ) return self._fetch_data_for_variables( - client, variables, client.DURATION, include_measurement_date=True + client, variables, client.DURATION, + include_measurement_date=True, desired_units=desired_units, ) def _get_all_metadata(self): diff --git a/metloom/pointdata/usgs.py b/metloom/pointdata/usgs.py index 12183be..8740bb2 100644 --- a/metloom/pointdata/usgs.py +++ b/metloom/pointdata/usgs.py @@ -269,7 +269,8 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], duration_list: List[str], - resample_duration=None + resample_duration=None, + desired_units=None, ): """ Args: @@ -279,6 +280,8 @@ def _get_data( from self.ALLOWED_VARIABLES duration_list: list of USGS duration code, "dv" or "iv" resample_duration: optional if we need to resample the data + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -329,6 +332,7 @@ def _get_data( df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data( @@ -336,13 +340,14 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data """ return self._get_data( start_date, end_date, variables, ["dv", "iv"], - resample_duration="24H" + resample_duration="24H", desired_units=desired_units, ) def get_hourly_data( @@ -350,9 +355,11 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): return self._get_data( - start_date, end_date, variables, ["iv"], resample_duration="h" + start_date, end_date, variables, ["iv"], resample_duration="h", + desired_units=desired_units, ) def get_instantaneous_data( @@ -360,11 +367,15 @@ def get_instantaneous_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ USGS 'instantaneous' data, which is generally 15 minutes. """ - return self._get_data(start_date, end_date, variables, ["iv"]) + return self._get_data( + start_date, end_date, variables, ["iv"], + desired_units=desired_units, + ) @staticmethod def _get_url_response(url, params=None, parse='text'): diff --git a/metloom/unit_conversions.py b/metloom/unit_conversions.py new file mode 100644 index 0000000..11b19f9 --- /dev/null +++ b/metloom/unit_conversions.py @@ -0,0 +1,133 @@ +""" +Optional unit conversion helpers built on ``pint``. + +metloom clients infer a units string for every variable they return (stored in +the ``{variable.name}_units`` column). Those strings come from a variety of +external APIs and are not always directly parseable by pint +(e.g. ``"DEG F"``, ``"w/m^2"``, ``"CFS"``, ``"wmoUnit:degC"``). This module +normalizes those strings and provides a single conversion entry point. + +pint is only exercised when a caller explicitly requests a conversion by +passing ``desired_units`` to a ``get_*`` method. When no conversion is +requested none of this code runs and the returned data is unchanged. +""" +import logging + +import numpy as np +import pint + +LOG = logging.getLogger("metloom.unit_conversions") + +# A single shared registry. pint Quantities can only interact when they share +# the same UnitRegistry, so everything in metloom must use this instance. +UREG = pint.UnitRegistry() + +# Map the messy unit strings returned by the various data sources onto strings +# that ``UREG`` can parse. Keys are compared case-insensitively after stripping +# surrounding whitespace (see ``normalize_unit``). +_NORMALIZE = { + # temperature + "deg f": "degF", + "degf": "degF", + "deg_f": "degF", + "°f": "degF", + "deg c": "degC", + "degc": "degC", + "deg_c": "degC", + "°c": "degC", + "celsius": "degC", + "fahrenheit": "degF", + # length / depth + "inches": "inch", + "in": "inch", + "feet": "foot", + "ft": "foot", + "meters": "meter", + "metres": "meter", + "millimeters": "millimeter", + "millimetres": "millimeter", + "centimeters": "centimeter", + "centimetres": "centimeter", + # irradiance + "w/m^2": "watt/meter**2", + "w/m2": "watt/meter**2", + "w m-2": "watt/meter**2", + "watt/m^2": "watt/meter**2", + "watt/meter^2": "watt/meter**2", + "watts/meter^2": "watt/meter**2", + "watts per square meter": "watt/meter**2", + # flow / volume + "cfs": "foot**3/second", + "ft3/s": "foot**3/second", + "ft^3/s": "foot**3/second", + "cubic feet per second": "foot**3/second", + "ac-ft": "acre_foot", + "acre-ft": "acre_foot", + "ac_ft": "acre_foot", + # ratios / angles + "pct": "percent", + "%": "percent", + "degrees": "degree", + # rates + "in/hr": "inch/hour", + "m/s": "meter/second", +} + + +def normalize_unit(raw): + """ + Normalize a source units string into something pint can parse. + + Args: + raw: the units string inferred by a client (e.g. ``"DEG F"``, + ``"wmoUnit:degC"``). May be ``None``. + Returns: + A pint-parseable string, or ``None`` if ``raw`` is ``None``. Unknown + strings are returned unchanged so anything already valid still works. + """ + if raw is None: + return None + key = str(raw).strip() + if not key: + return None + # NWS returns units like "wmoUnit:degC" - strip the namespace prefix + if ":" in key and key.lower().startswith("wmounit"): + key = key.split(":", 1)[1] + return _NORMALIZE.get(key.lower(), key) + + +def convert_series(values, from_unit, to_unit): + """ + Convert an array of values from one unit to another. + + Offset units (degF <-> degC) are handled correctly via ``pint.Quantity``. + NaNs are preserved. On any failure (undefined unit, incompatible + dimensions) a warning is logged and the values are returned unchanged so a + unit hiccup never crashes a data pull. + + Args: + values: array-like of numeric values + from_unit: source units string (raw, will be normalized) + to_unit: target units string (raw, will be normalized) + Returns: + numpy array of converted magnitudes (plain floats, never a + pint.Quantity), or the original values if conversion was not possible. + """ + from_norm = normalize_unit(from_unit) + to_norm = normalize_unit(to_unit) + if from_norm is None or to_norm is None: + LOG.warning( + "Cannot convert with missing units (from=%r, to=%r); " + "returning values unchanged", from_unit, to_unit + ) + return values + try: + magnitude = np.asarray(values, dtype="float64") + quantity = UREG.Quantity(magnitude, from_norm) + return quantity.to(to_norm).magnitude + except Exception as e: + LOG.warning( + "Failed converting from %r to %r (%s); returning values unchanged", + from_unit, to_unit, e + ) + return values diff --git a/setup.py b/setup.py index d30a058..ea5144a 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ 'beautifulsoup4>4,<5', 'zeep>4.0.0', 'pydash>=8.0.0,<9.0.0', + 'pint>=0.20,<1.0', ] test_requirements = ['pytest>=3', ] diff --git a/tests/capture_unit_fixtures.py b/tests/capture_unit_fixtures.py new file mode 100644 index 0000000..1bd589f --- /dev/null +++ b/tests/capture_unit_fixtures.py @@ -0,0 +1,232 @@ +""" +Dev-only helper: capture the raw payloads each metloom client parses and +pickle them under ``tests/data/unit_fixtures/`` so the unit-conversion tests +can run fully offline against real data shapes. + +This is NOT collected by pytest (no ``test_`` prefix). Run it once to +(re)generate fixtures:: + + ./venv/bin/python tests/capture_unit_fixtures.py + +Strategy per the agreed plan: + * Pull genuinely live data where it is cheap and credential-free (CDEC). + * For everything else, source the payloads from the committed real-response + mock files already in ``tests/data/*_mocks`` (these are captured real API + responses) and re-serialize them as pickles. Payloads whose canonical + form is not stored as a file (Mesowest / SNOTEL / SAIL) are reproduced + here in the exact shape the live APIs return. + +Every fixture is the *raw* payload (parsed JSON / CSV text / row lists), NOT a +finished DataFrame, so the client parsing + unit inference stays under test. +""" +import json +import pickle +from pathlib import Path + +HERE = Path(__file__).resolve().parent +MOCKS = HERE / "data" +OUT = MOCKS / "unit_fixtures" + + +def _dump(name, obj): + OUT.mkdir(parents=True, exist_ok=True) + with open(OUT / f"{name}.pkl", "wb") as fp: + pickle.dump(obj, fp) + print(f"wrote {name}.pkl") + + +def capture_cdec(): + """Live pull of real CDEC data for Tenaya Lake (TNY).""" + try: + import requests + url = "http://cdec.water.ca.gov/dynamicapp/req/JSONDataServlet" + out = {} + for key, sensor in [("swe", "3"), ("temp", "30")]: + resp = requests.get(url, params={ + "Stations": "TNY", "dur_code": "D", "SensorNums": sensor, + "Start": "2021-05-15T00:00:00", "End": "2021-05-18T00:00:00", + }, timeout=20) + resp.raise_for_status() + out[key] = resp.json() + # sanity: make sure we got units in the payload + assert out["swe"] and "units" in out["swe"][0] + _dump("cdec", out) + except Exception as e: # pragma: no cover - network dependent + print(f"CDEC live pull failed ({e}); writing static fallback") + out = { + "swe": [ + {"stationId": "TNY", "durCode": "D", "SENSOR_NUM": 3, + "date": f"2021-5-{d} 00:00", "obsDate": f"2021-5-{d} 00:00", + "value": v, "dataFlag": " ", "units": "INCHES"} + for d, v in [(16, 12.1), (17, 12.0), (18, 11.8)] + ], + "temp": [ + {"stationId": "TNY", "durCode": "D", "SENSOR_NUM": 30, + "date": f"2021-5-{d} 00:00", "obsDate": f"2021-5-{d} 00:00", + "value": v, "dataFlag": " ", "units": "DEG F"} + for d, v in [(16, 33.1), (17, 34.2), (18, 35.0)] + ], + } + _dump("cdec", out) + + +def capture_usgs(): + with open(MOCKS / "usgs_mocks" / "daily_response.txt") as fp: + _dump("usgs_daily", json.load(fp)) + + +def capture_geosphere(): + """ + GeoSphere data responses are GeoJSON FeatureCollections with a top-level + ``timestamps`` list and per-parameter ``unit``/``data`` under + ``features[0].properties.parameters``. (klima_mock.json is the *metadata* + endpoint, not the data endpoint, so it is not used here.) + """ + payload = { + "media_type": "application/json", "type": "FeatureCollection", + "version": "v1", + "timestamps": [ + "2023-01-20T00:00+00:00", "2023-01-21T00:00+00:00", + "2023-01-22T00:00+00:00", "2023-01-23T00:00+00:00", + "2023-01-24T00:00+00:00", "2023-01-25T00:00+00:00", + ], + "features": [{ + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [11.700833, 47.5075]}, + "properties": { + "parameters": { + "schnee": { + "name": "Gesamtschneehoehe", + "unit": "cm", + "data": [3.0, 18.0, 22.0, 18.0, 18.0, 14.0], + } + }, + "station": "8807", + }, + }], + } + _dump("geosphere_hist", payload) + + +def capture_cues(): + with open(MOCKS / "cues_mocks" / "daily_response.txt") as fp: + _dump("cues_daily", fp.read()) + + +def capture_nws(): + with open(MOCKS / "nws_mocks" / "meta_and_data.json") as fp: + meta_and_data = json.load(fp) + with open(MOCKS / "nws_mocks" / "initial_meta.json") as fp: + initial_meta = json.load(fp) + _dump("nws", {"initial": initial_meta, "data": meta_and_data}) + + +def capture_norway(): + with open(MOCKS / "frost_mocks" / "hourly_temp.json") as fp: + _dump("norway_hourly", json.load(fp)) + + +def _csv_subset(path, n_rows=48): + """Header + first n_rows data lines of a csv, as text.""" + with open(path) as fp: + lines = fp.readlines() + return "".join(lines[: n_rows + 1]) + + +def capture_snowex(): + _dump( + "snowex_lsos", + _csv_subset(MOCKS / "snowex_mocks" / "SNEX_Met_LSOS_final_output.csv"), + ) + + +def capture_csas(): + _dump( + "csas_sbsp", + _csv_subset(MOCKS / "csas_mocks" / "SBSP_1hr_2010-2024.csv"), + ) + + +def capture_mesowest(): + """ + Reproduce the Mesowest timeseries response shape (units=metric). Real + responses nest observations under STATION[0]['OBSERVATIONS'] and report + the per-variable units under 'UNITS'. + """ + dts = [ + "2021-03-16T00:00:00Z", "2021-03-16T01:00:00Z", "2021-03-16T02:00:00Z", + ] + payload = { + "SUMMARY": {"RESPONSE_MESSAGE": "OK"}, + "UNITS": {"air_temp": "Celsius", "snow_depth": "Millimeters"}, + "STATION": [{ + "STID": "INMTP", + "OBSERVATIONS": { + "date_time": dts, + "air_temp_set_1": [-2.1, -1.4, -0.8], + "snow_depth_set_1": [1200.0, 1210.0, 1205.0], + }, + }], + } + _dump("mesowest", payload) + + +def capture_snotel(): + """ + SNOTEL data comes back from the AWDB SOAP service as lists of + {'datetime','value'} rows; units are inferred from the station elements + (storedUnitCd). Reproduce both. + """ + payload = { + "tz_hours": -8.0, + "elements": [ + {"elementCd": "WTEQ", "duration": "DAILY", "storedUnitCd": "in"}, + {"elementCd": "TOBS", "duration": "DAILY", "storedUnitCd": "degF"}, + ], + "data": { + "WTEQ": [ + {"datetime": "2020-03-20", "value": 13.19}, + {"datetime": "2020-03-21", "value": 13.17}, + {"datetime": "2020-03-22", "value": 13.14}, + ], + "TOBS": [ + {"datetime": "2020-03-20", "value": 30.2}, + {"datetime": "2020-03-21", "value": 31.5}, + {"datetime": "2020-03-22", "value": 33.8}, + ], + }, + } + _dump("snotel_daily", payload) + + +def capture_sail(): + """ + SAIL/ARM data is delivered as netCDF; arm_utils.get_station_data returns a + DataFrame indexed on datetime with a column per variable code. Reproduce a + small frame for the precipitation variable (mm). + """ + import pandas as pd + idx = pd.date_range("2023-01-01", periods=6, freq="h", name="datetime") + df = pd.DataFrame( + {"accum_rtnrt": [0.0, 0.1, 0.3, 0.0, 0.2, 0.5]}, index=idx + ) + _dump("sail_precip", df) + + +def main(): + capture_cdec() + capture_usgs() + capture_geosphere() + capture_cues() + capture_nws() + capture_norway() + capture_snowex() + capture_csas() + capture_mesowest() + capture_snotel() + capture_sail() + print(f"\nFixtures written to {OUT}") + + +if __name__ == "__main__": + main() diff --git a/tests/data/unit_fixtures/cdec.pkl b/tests/data/unit_fixtures/cdec.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6095c62b8a928399782129cb881a194bc2b174ef GIT binary patch literal 941 zcmZo*nYxIX0Ss!VX!I}_m#0pNouUC^a~78*mSpDVd8SP1VGi+&oYKRdQd;DkpOQMI zhtXw94_C0OU$B2rykDsA6mRApuHw|Z;{2kJ$^xJ~d$6B>xI(z|lpdCp#1fzYzmb8F zp{}W}p{as_ft7&)P)UALu?r44*0RK$($p#L`?qu17A(Hb)x!ZaCDAP>F&${N!jvA? z(!9));we3Bo_@|AuEA4Ef$q&noV+Pk3T+YixVnlsG8BkwPPLCIy&hrId z{qgvpZ|;cX!`5dHPvBdG^cs5@+WtPk(y;B5t|=@#Wi}IePl$?cEn&gOvxL zzbFTN_x;c1aJU_gx5Mdn_~hlQAAbVL=II~5-hK1-^}Ao2uUh;}cuiyXf%RGMi{{9D_?QwmN$Mr>VJm>p8{^r|H!T9Fy+i`JLTnc}eOV!`l{`Hf; z3|-$0-7q#w*YsW2JnF}GTbgl-qhV?Kwb|B|ze_t$O+PnFyDSTz!@f4dGI(ikyV(H) zpAP7Tp_$uxnHVs(d=Bk6HeLMlsB1e$`)A_S^U$vO(#`Ky?tIj@%fJ~o&Aos2nJ>q7 z*!re#r(tcF<23LzMs|G3ef#d`Hs&Y8yLQ2{uI-tVHet!^i-vaIm~gX=?cQ9hAHw>n zop?T$deek$-Pr8df>q9~AG>A%pqW@?v} znXY&Ps9H?DkpLHi6qe&^EblQ-|1p3t z=mdF$6U{Kq&9*=CPWD31}6e8kTier@olh6U|~28v8^b(LVNK=mtD zdEYKq{YR(6C(WM2?2%Zym>u?x@?Enz(`-o9&W0i<-YxB~X8Vk7OxmA34u8`W9yX*Y z0C@HId<*)Ch!LN8Zzjlz*?L#9Ooa8`60dtJ^YA!^;U>)A3@xv3E&h?~x|bV5SG*!y<4n!OaF zLy0-Yj_o=&f{|vUc3v5@$PeeTGDbk4BmQPI+<*)-?#7M%%ImzmRd;K=o=$;FkF6@3 z1kcFkl8%N1Qkr+*Xf;HR7DtD?auz!wVBy)qUWBk`vlSR!|ZU zgm_#Qe;MI9s5iNC<5>b_E&_ z35bf+Ew{`oF@>0$F3`#1a}dT5gAjJK;#ApcGNeJD4d8t z3j%&ag20Z5!5jk`mD^CQD=VmdBpQm1?UVokg&_oiCIf*~wm}iY0=lI7t19vVL-8l- z$gHt#;sY~eOplJeJO^)xEl^(IC^!^Uig4T(4Ln}E!r?@^HsN*n z@gSi9l#c}lj0kv`q*6srTqt^g1nw}6kq9DoP*V9nAdy%^jspvH$D0{IQ1B|S!i57S zTEMgzF|CW>lBc(34#9{*oQozEmtjr;kDmd-CPStG0#5`4L)zIeC1i@0_Dr4ob`#GL zE87^fSzvY5(j;-UF(-_ZaN$)$vZz$3c#BfLM*$^J&;Tf40QOooB8KDyBo-gTxM^Zt z=yhHm*DXD;-fsuiS;Pe2u=W^z&cJX$9vO~$Y=E_3$p8@8!>6w{$MQ!(<&SX0pITj~ zpr#KnzbHt7qey8y1tWRZLe$>S&}ox78;=`tTtw&A2WVui$(%rAc1nbZgcqfC6AtXc z=ozZvWK@ZKVBJ7=AQqqu&RR_xj<80PCYDeGMFqt!=?0ZRN#UzCoC?cBJs*J*oZ>S| zfw1L>82l|Fr3}Sf3BuGOdGq2@d-rQmkoZ%|&kaEn5og0hY~)FWidsRIl?NB_DHDm8*($&nhdoEQ@SWtqIukC|RrSzIAbivx57Z;!pyUfUZt+qK zQ=^!-Nh6HZUsp`BEae>=h&eBHWP{T|4X8kg&NX5lS-_L+NSf)X-}#0zBv$8ez!3^5 zaYFfu`Q%0t3&o2Cxof9^6)eZZBn;+0;*DJGnhhkd(AS$^oH3yvNF0Cw&yiouGBnO= zK^9?hsA;k!<>M4Aa5KzEIE=cD{6dNKVPi_x!Dzg>N3dWq!!6)2nhHI@G;fe-AQDeT zGFWp(da)c8u)N-(|714GNh@&V6zPo!=*piAgf3I-mS>hN!)WnmiJYr2t1MS=+{|=J z#9+&zAoeYIgKaMcK1?weP-q1e1&JadjZBK{xC=}kP-L)F6BDiNP(P3H&(MJnIno$l z9nK{t!5m-%dP2cXG(mfwXB-U_qjiW{fWdm_khitk>d`!BH@g zell_@Q8ID_G;;S5IH4oNT$NSk1;36Z%mtJ4S`OAQPv07zr>eqAlJPZeD5wfiRD3dg zgf#AUT^^y6p!5MpG#XOvH)bg)NTJ>prWhiF=rwt~QJ0ZcjEK_7nzL9~4QIgO3`LE) z4EzGR%8(>TR;f!VLUf2ZFczFakS=L;L`^_*)F3xBP$^A{UR5kN(ma#hMMZ|JRG|e< zwi>Nyz!A`pAl3?$VMYyeg}Bk>u`ILOJu_saOhGyfsXcULwU(y}Rd2F8!Q?SN7o%e~ zc}6&EXA^%CE~t2%EaHW0a-+Zj60zVDk6M$V&p{5{ZcP^1<)e5|fMB4^OgyCQ zM`R?+)aXvIPIqn^MO>R{EG}7+P{0rr*i0~yU8vF^LuWt?)DcEiuWchuZgk4h76sso z&YW##i3-X#XQ;zun}V=H1o$(P!I>xzdqg=yz@1@9RJe3PMUpIt9H@}8f+c;f;|`Ya zV8Qu_xXO|-!jqV(W)jF}mY7|AEW!!hcnyI@WSLnheuxQ7!+&~b#V;hv5C%HEgbBRy zvP`tL;}ubcv)PfYo!ELX$6zSzG?(xP%=j|`4DD!`_K5CK7TPtv-s9?-M2El~N*{)J5O|XAqwRJdrb%Zw_f zfkR#87B~V7!W3j`xFi-4Ye#_^XV+#BPj!o?#EMJy4T4_LNaaAx)HJD!G^jbTZRGC9 zYWk#tP>cMuf)Ec90$;f^VlOU7%y`@p&46rJ$B>+zYft&)pCm*luc1el;$wteHq0q# zfJy-g#EzC}jNc_!&A1xsRBLu&&fIr2%;PBsKP+~#Xh-t#o5{-Tf+3J3R%sW8q|CCJE^wsu4O1Y`iFf*$n0;XS_tj8D8eMECHGsO`$4nIVd zM<}W)_e`uY&tq*@2$E0)@TFsRUg*+xaJ$8r0cqEmc8QIk6!uH#*d z&{g6q+{0Hyi`BtNLV`iZu1f@&(mvak1BwTIG8}oMMIds9f^wFHwY^%PV5$r9Mn{4f zUG$@FyPyhz3cg04R=iGE2P2U!n&%O+vuj~|3Po71;d+~X6e!dmtI-TgjO-NwMotu6 z?gj?5aCe1E(*a7c@6KZ;tl|r}3?ocEx5QFFD3ZYl84)lfA!>9a?e?tj+zbgcie%h+ zAAqa@sEoZ!4WpS$?Ri0=u535cAF+drv4}h=*=GWL!{N9PN`XLxDavZ$9f}Eaa=@By zw+(`i1xoJGIsz$H)jkecHfP1LMJw{f)KQ6ypV|U4fD8-~LbHoX$RM|LGJsP#y{2>q zhzAZ4F6SO$EN?ca)Lw z8c%4PyDuD;0ak~cqAsxr^XDW04+7Y3B^HfB9!2hD8+(hLe0PuuALLIq)U;aYKrnDy zh6MgFD46c8cU-hGllPH8LXyr2gskCOo`*_9ZCmxzg$%LZphnv13Uz1i0mpYlE2I6T z0Xc;F(m3y#wJ8G-AhMoqu{!&X-b=zII{`k&(0~_OiV!zomMY68;NHOti?@}Z`>=`x zAEPz`{8*p>i-h3I6Cq^fOtMjSYQ=>%ONd~AT^8e_c71MffsheToqEk|wG^W;?}1dZ zfgsHYZV>1#k!<<#TB2F!LSbFl_bjzR17eUr;1|p#+y)&)|JQpNVB&DVOb#Svw3Fj+X#RGM_EPy3@V^9xU>@ta|e-20f%Yy6F9_Vm#qMk9pi|S zGGOEeUQMfbQ(CL>_+M^sSNHT$q-TVygj7yU`FK7!L1#b z!6&QbnmAyl%!sJi2!vopAbYMS3|vgGr(#c&L+SxS_n}r~>4_R(#7L|F#%}kJMA(Gm zLTI>TQ8~MWCHN>w-H{XvlSi}+N5Fy8;XvCQ>@+fQ3}) z#9DvN0h-WXkB9e3slXA5fZ08Sqi(QR&&|>a=JH4XS4y)P$h%SSzgbl`2-HpA)vygaN$P65h9=kz!>ZZFf0vK(cs9gQUZpN z5`MCDD^Qe3NlrLih7@Wbv}KKE*AmkrC~Zf~)ZVmP5TR>E$$~RVLlSIpxkJT-D$$AjIiW##qU}Oa4gllNVg|i%esebK+)xa%V;VsSYpMQ z`Jn!)qpr^|z@$>R!T6L81ikimyha_FrCN;@H&Kk0+?9PE%QB$fbTG=Pj1tT0xBAdy%qI{-r+W%NOdgB=GsGQ+M?j+jx9 z&2zZ>(liIItUk$?y2}J_d53MLmW>ynz${qv+^zs>K}X0~kQp!lK`|r}ri%{=fmIF>EQ)@P?oS>~l0DFEgt$;=+ zmvIL+0(Qd&`^5%Vv(O4`NV%yf8F+g(2Pr25p?D{Ata`!(xp!x|6%7s?y7L(P0_qn< zVF8iQ!23Kz2k^8-Y5qnGt1-{r31kOC`?C|*Y!(t^4(4Tl!Ua4U1a{;CDmUTbx6`q> z%bJnG&AIl%M2OiIBi@j0fgydlu=f%&6KZhOtHQ-&bD&FLkpW4x7N_v88cc!3(K!ya z-ab^8XkydzmagHFu#J_;1FbE)E%+59Z0;C*7{}g;Spw52cYz=hD?ev64SPJ$!mm)jHKVN|3LHlk9Jq}yDs2RopE8jRHd>0YPe zL3U4_Sk0S?u23CJNi0hYMX}|U&jpoGnfiOF!W3p+gGe=Vs<@t}H7xMwkGKvAy~a=$ zun0NqKuaCY%&2xrS-8%Xi$>G=!RY*tCw9{f8!r((OguV4Y|YqHTp$I}MTDp=NKWuh zSVSCcv26%-6<$lyh>T5zie{@VcDLpfm&64F#Kq`hj^_wc1d*qzyNN;77D;JZ;s%m4 Xz9_(?&qr~|-@Iz5dxJDX{@wl`ycT+s literal 0 HcmV?d00001 diff --git a/tests/data/unit_fixtures/cues_daily.pkl b/tests/data/unit_fixtures/cues_daily.pkl new file mode 100644 index 0000000000000000000000000000000000000000..45b93ccf05160e7b61f52ef2c7b3cbed3089e0c7 GIT binary patch literal 213 zcmYMqy$ZrG6a`=hC!gYu3T;eN`=ecT(@k&_u5FNDnuH`!y7>T(`brKym0FP;&T{z9 zb^LgZez`q&9Nii{-;0($r3Hy_rK%_fX88q+jRGN5!dG+JgPUj)g6lsPGjAVrX* KWQ3)YJ9z_vo<7O| literal 0 HcmV?d00001 diff --git a/tests/data/unit_fixtures/geosphere_hist.pkl b/tests/data/unit_fixtures/geosphere_hist.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a2f4e27d7b35522bcb233908fc5b29148975cb61 GIT binary patch literal 515 zcmZ{h%}N6?6os{pcBmV>@fB3Gnoj*Gf-oR-wk}*M5@v3lNGA#TDRd$90WP>n-$#4} z-@?7%(#%X-bTL_kllz66oO9>F;%9TCTCeU9I|XEnKAD*WhkI1Y2}`MAf}g!;!5s!Q z>5hOJtKe2lCXklgVS5TnSE@DbJM0)%fHt&{+CBQt`w@u-J<{(*R0x$ZP7PGvlK5c! zxw!cnC2{ip{qgp21Qt6|2?@$Df2T@TQbm0?ACf=gj!tt9P@#Z(;INy3riH0NE+F@{ xz;fnaY^8z00y;FGw@su8};=KIw_>|Owl8h-m+`gGPIhnbs zC8K|K&^&n3dWifbbBK4u?qxz~ScZFzE%1aRtmsEiFwf)dK+IWOnBO literal 0 HcmV?d00001 diff --git a/tests/data/unit_fixtures/norway_hourly.pkl b/tests/data/unit_fixtures/norway_hourly.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b360aad25d0f93fe9f23d10370f41e7485ed2106 GIT binary patch literal 16704 zcmd5@-EJF26izEsNSn5SfS>{uL1V}H@%}nZYbokR5twQt8WD>NBosFG*t4+qy1TPh zeSy>mn9Jn?xaJvn2@p5D0TQpkEoawGunUssNLfx-Q54VG<8QumX1?#tY@V0C+dWvy zuW!?;e9dWx1Hqn&RKClF7{(1)e;Z+V)qXx#GqENwpe{l}mFIQi>0 zx!O*mhz-PJ!TN_`G7zczm0a=RRea2%W3R`uhHGKB8#B1KByWjOc){G;Z%q3*@(16f z^5=8xbDV_#P!Q`QL;E@o2SMji941kl#rr3p*7xq4MkQih25{O;0L=Y-9L9s1VYUpT zY#*HevOd1JvpaXE!UEQZ(Zu_nADzM=hAi?#60u6w-((Q+eus6vBoLK=jaYE;GL@x{ zC%p8_w3XVFi5hrgzq?Rnb5wHwV$xaXE9)ec&`D4-7xBV z5R^ktuwEFAQ+eg1RIajT!!QPNXPT{4-gugLfiK2~VFz+&De)2*IDc!FMgI0YZ{{p- z{7z04fA8M}@w-i@{7ZKI3gerw!avBmzbDuJn+0CZIpCG)zI4SwuRmSPwBq8#?GQg&@Ntl>>2%Ou5I3FNz?%4T7 z68DD)oHaU}=A1AtSl~56oa#d4-lhn+iBTY(Ze=$y3Pgz$qd*#+^DxyFsOeDR#0Zf_ zx4@g~Lgd~endle=+9pg)i~?=Z;j}OcwE2B@{Xh|SOI?AQPL*x~S?WUMu8ke_pD3JMQ z?lmvl$tmH)D3Dc*QYzt8S0MKd5+c>5r|Fo_^W7s7oEQZ%idm490!lLFM29Cq{_KoH5|U2$4xQ(Q&2vQMSSpY$*Vr>O$nc{{L4R01%@<2Az{L zF$zSuVj|Abn@){z0;wy|<)8PfNDn-wJ@e@Bf@8ofy2M14>Vzd5s#M>gDuL}5s#GuN z_*W%AElep?p_=G*T&Z5BiaX%MI3%)?4RB%{(mElZnEYHP%ujWP)O0BM#26(?J~2j# za=v3q^$L}~!+cjaN`*atwpfn$p?(f}Z+N z3OGiH$j-8WPmB(f$TxDiF5- literal 0 HcmV?d00001 diff --git a/tests/data/unit_fixtures/nws.pkl b/tests/data/unit_fixtures/nws.pkl new file mode 100644 index 0000000000000000000000000000000000000000..df1ee2bf0d6d643bd2c8a14876902fc4756f4f37 GIT binary patch literal 65997 zcmeHQ3$R^Pc@BYuBm}}UAfkv`3_`e%BsUMx97td%93m)JK`b&S_wL+#;NEjC=aKNJ zmex8`r$=n3$7-L}wy1!W+LvwZKy0VdamM=Sw6>Mb)Z*y0TD7&d_BHlf>tE-byVv@@ zbMMtSwJjMOA@~2*X?|mOUr^M>x0&Z*T}wNqLf-A1dOd8~ z4gA}>nY=!qw;K1k#%9-a*4*q6-Og!K^?GBvTC243X7`TnmK)ykt#5s+e0kpPY*=X2 z7xG4{iXXTEoLpMb2-cKr2!u?|s#6%? zi`3NW-k>?+VQXg^)#-!qgtfB9{pW4n@{2qE-(K4-AMC8_wcPFOP;EL`a2c8%**0;- z&TS(DTz_x}>g~qdecjI0-&R8-vjKi)ujvf^?yt1qg)3IlVz+Zn{ctmH9Idpf_1Y~h z|9{Pu&ac_Wo!Q9e(8kgH#`=_RE0Cg3m)TPbEScQ^W@N|jc=)GxU-FhGveI?G^~UQS zy!umldv>E^#wkyORX zUyc9TGY`bk%3M3|W~DEE{R@vj{@kqHIrVT}X;f=7eqbY=%{}Ake5R4-cW$ZFX6EwC zy0c^D&$Qd^&c>dDZ?C%%Y*o&^A3tvP13hxnt8O2i80&V{PSx8rH-*cG1I-MueWT+$ zx}8l&s*U_s%R=k;vf+4V?>e2>HUZDHs`I(Q&bekjZ&ar$BRAHYcV1JQ$>)%BosEsk zbfpoZjg&6$yY{v&CVR@XV~}&F*Mt=^X$fW6*E#j(`rLgp^;)70-~j#s4UT`{vUNMx zf(8#x{NPtU{zxAF_O{=?|NgJxZ{P3(s~`XC-rqj)z(Zet$GrR7Z+`Z`kN;$_ed3c} z{QQHNYvDIuciNM$?6nZJn^#g5+R$FW^z%vhuG@Le_~`idt)mlL$9GJQkL?*7-Lrl7 z<)fo}Mn}7y4d7ID+S$RCmn;rjy@_6xfHtYNI?slF(WZIKs2fDMj zXQk~A?Rw^+U7v3QJ9}3VF;R3>53%#r{trX!b!v>jrhtQ>>3|0`aK(^H_qIm>iA!@bM@=pP7zF zjfc>^%#Y?GhKPp~5LdZaKl2bnZ0CLK(tSjE)N|{E2py&@2(?2yNf1hr7JX1cmCt2U z7EBr!OqzgBa3_t6i~xd0IV+9HK#YlCk4h(viC~WkwqjHw*hA(ag1tlcA%d;c1EZqB zEaxkHY;qDjPH`_p;llZ;y!{FsLTdJsK=P7B!HkyDyK1nsi4qh zqWS|uP)H>x5JEN-v4IdwQpp2^VA4fG1a?W;*erwwh?q%XaoHq900d|V5%(u9n8A_+?@-Mc1}5~PO^ z^VmJMA6|8B4(oKa0nW+tRz-0hY+C? zdjy9Np%a=0gwh+G5vs5etGw$&Ap{d??O_PPM4E$!piGPt+?y)(p!*O*#Oe_OL3dz= z5W&8X9m5zx1bfJ+M6k7!lB%tWhQ))co@emJUjQ9^~rY3PVQ( zLc~lhU_gkNsYMb95i_+AkISM(Wi}9T%>0TGpL_V2scMDp*$g2PlYwrbDTd zU>d86O+ZEfh*&DN`}+yW3wdLGcI7Cej62gf5CNgI6$tJmDH}j= zCzTXif#6OmDWL&^Iw>30N9-#L-LQG^%pT@ln%QKLlULD0cp*}HNY?R`9+pE%W*a@^ zB8RI}k#%a31A-Gu&I5>;P`mxgC4D8YPG#tTi2DfiP>BIVOxmziD4GMYLJ9&QC^%$P z%cUUh8e=U8Vy2`C2qDUPI4h-~yaAOI#5J0jhZ+?S@jNVaQuu+0No&QEBrOneuq~B_ z^pR+@jcUlulj12vP$eHOyyAH*5py6!VknjX5hq4iV}^`sg)~PWl;)XCa9nuBED7bW zULcT|Ehm#PQ4>onH(B>gA6WTn47fE>#CP2A|V-Z~9#90>*w6ZfqoI`_mJAu3cuh{Vv|Rf%yH+X`DBv9Bnh%vA83Zfp%9_^m^i@Z0FYCD!+m z$ap}|<;V~*p*$ik%FUFkF<<2k!DGhPY#?xE9%qCxiJv?fVrg1O5IX1dn-J_K8N ziERizD!u|6A|BNe!Nxf$^lY*466@u3w)9j0aAP)BtXEBF%6*u6PXu!={dzPFt7*L(miOYdyOv0QfE4c; z`w+xVLW-A}0pepPA#^T2b`ny|))4HT47S`RcTL$K2gL<(*Bv7hv&n z>{O1g7_8`Cio9Z}E~LsA#Ff6gyP`Em`8;OIb^9!48W(R?0h~!pd3jMjotg5w;xuNO z1jMb!<&DfV)@Ccv(pHg6d;sZKS=#vlr0;aVG#-FCEA=Z(6Y=t`rqP0Fypmh1=O-S3 zX4nUc(}ljZ%jrU23FdTp0U1DYWMgA}5i{k{x|Er6x?ILgd9+^2OnJ0ut2|nC zT0C0MV*}vPVu9u7p5@J~RUR!yLXq5OlU8{dR!${Dy=R&}xr^107hdHvk^x}Vi?<2J ztTee{%9)iWH(N`6#-_;)Q_ie&*jPEU(wB?TDrZ)jhA`#KdNCUSXI7d8XqD^0m8@0H ztZ`y0bYLel<LOq2ObZh7Qm3V1Dnu~l36lyD>QkQF>4y|x%IQK|jp<@zO{axc`B>A16Q-P5X|!O< zne~z+urTG!+R98hv&NVyXVwm8%9(W)F%`*u4KwA@%9tsS)(y;*N9%fK%A@siX3C?* zoW&2v%Pd-=f=|mFb-7t)%;M*SWyV*r$g?aEi#*FMe#lZzS3`AySx(njY&v5dFo91C z61G3}D|b2om~y(Lngvr%mvrTZDW^-ia>JA-_X`se!jva>y2!(nCwIy$nDXRKg$QRh zaR^K_FDGN=54V-m)ex=nGIgrSkGvyhZh^8~;97qzz znevRiBIc_zW7O^Zj11y*G2_pw%av=|AKj5U3|n6xUE86XV~spurkq*P8f&JUS<%+aOgXcn#mG!Kv!dOxnQ|S7 zDor!x%!+itn=VfW%#=s#x*}c7lt=3TGv(2ut@3D3nU#hBt#W3ik^xiBtduS=<;;p!7#l0sft0Ukl`|_U9ek@o2cm`3OnJ1@G{jv$ zkJhzG8p4!EE82700C=>bb3!xa(MqKl1K`n$wma4;k5+WIVWvD<>AIyJjEJx8e5*nS zqFUQbd9+v>ihAG-5-o8|7ago)0P3j+T-)LP^qk7VLZyKV(i3`TE?%&)s=t7Vk4e)iOMl(Qe}39;(TbnA0)l^S!hNth&ff66 zL-7fb!OU^%NXe#01w`D3UnS)CaNNTWR4Cxx!lN#QQBl3fO2tP#3J5A2_(&OhB+$R| z#s`bHz32NHy$Z#3Imej?cPZF)n9V2cm>xHkz! zu<4uG)`tl8ka>t;i_gipQLRu!fS|#S-?O#}&?pb%w^ENikGenX)82`C_+uTdH|l5T{mcX+2!tS%N(>+bq3GvJtPe`4@73=B zi#G<~tK@JgutdDa;1D8$y~ncRxBuC5i|!0jf6x;M5kc|cE;lL>!Iz301cFA8Jn;Zi zAqh%1fl&I-zQ5uLl}JKZh&9c{4l})fB^SV9!=c~S|#8y|r*t&@g3~n$fcS+R+Q$bp_M*u?TjjAMo5PGBb2tWwZs?Y%; zNUN3#AOvZP&yo)k90);LC2c?m($bG&Au)guq*W_0E&kbb0s1`h?uK&3V#N?U28R$Uh7c+s zRGFg_Ly3r3^IabzT}9F25OKN&*`W!J6ZU7n@f-izhO9)_O7oRqBCSLWA(%*W&=7)& zG|xRk)h_f*!w_`gh`(uIrhm#OM2n0RU@d%|JD#3BpEJC-5-Jw|@ zG;~C7)J=X?QtAN&jamQI=dc5_anN9ib8kb4n9(aFLx`BsV%89G%mPi72&l_`abV*6 zP_m&?`hV!-zI5k;38)7F1{Yg+9^m|AdF$Ht+~I7lemFm&&jiHjjGd}_r&|B+Q}t?f zR#Nw>Ku~Ist+sv&K-9eMhF+X`h?>_S;>-`~LM7NcDXB*Tx7Da{`7B07vjKeo5s%9E zL7W6rDn#oxgQxT`3i7=v<#&?OArLxw z_!g5lzLIhZ-oeeZ$YmvXK*YQVdO?eB>qDfpI2__cDf{ zq>JkpGZk#3dqYErTvn&lm~x!UAp*35^nIublI}kZ5l1=Dl-8VvpfStT_Xpjbx(MBn zd5F+y@2aybTA$lILFax?t@`?2DL!u4MV_iv1&BC;UcX5W;nsMCjRO%k9w?|L z0E7?$E`UelOq*#JU>PKf{zB6LawfDob6Y*j^r&In#K z4Ev40N(5BhO1?!MzSpIeUR=z^F%Oxo`T#;?wkqO)5G+w*283XViU$yaB`W4X2$tvz zW|iPdM9@c^;31>bl85_Qq0MUP1|p8&5=%D_A|69VMdLw~7LAKQ=+N(HqL*Yr>cPD^Z?IgMakNtme!9j$vk zLS;5p4mQC?&vm32S}deEJ@AC<~%x)nA=9PE(U8y1dvX$bSx;=KbPVyO-LyJugVj^m-PL)C>exeoQemqRfZ*N#G<^aZvmJrviUeZO~f zJ#nAPC0fD>WOjf+1d|K|yjDs$42%jw6qpn_7 zlqJWqVcrw+U_FtX9zq_hCo4-p(C3LD&p4c<<>~kM>r{g{U;;^~y)dgd=;K2b2Oz`} zMxWmHUR6g1isR(?Fhow}5fCu}dk9uJ`cWl(8lK%a$l&bw-|_NGpCoP`2}%5kj3CL7 zQzEgxIE2_N2wDN5O3i0qRj`U81P~=N&c~7^V5d}D4^YmfC5RzJKGAR(Laby|Z4D7m zBV-^|5+JY#6{9MyeT$!_RqJbU<^$)Vk9bsL6T_UVw^fwuei|wq%2i}r<>f72n5=OT z2dbThNM>wjT~wceSk{BC>yK4y)B9$tb8}1W*)AFKYre+zLrSFkq2qXxgs$2869GWP zEDF4%XCKyw@Q}KKL?4O9nooBA#&w zT=BRy_{b269Ox|=0OWi&gn=fvM<^euX<*@sLmj5Ei9G~veNa~sucGl~`DhSiAQB%r z$?V@`4crOSL@NuF!;#^-=H2)0s~^o9-|6qGvu6fZvGc-MP0jRFVw&tu%ryOWKF)q( zs+(!_MN{YpOk)?DY4n{`GmQ_V{n}Daf0+*sdg2)C0VH8C)95RqegMg?(M;1{K7|jl z#k4fj=+mEQ6(<|qS#?G<RpH6279e;C@}pOu1jzFjMZ=lbI>^>uO>u{F?sKM6#w^w9<@)DNpX`^REa2 zIm)tDQ@@h)B{NO^3RCXaG+|)M{hIO>rrfX52XHYhR1RDRqVM{eX)MR4F432`y{Sm< zPs{>Q_?gC=bAtJL{yxujD*IwMzE-W_pVIy7lR9lfS>( z$fsJ>dM%V%n|r?+csieHp!T6IICM(t zS#tAa3_BySbrl6&&}J9 z#M$m68hxMN?YIkvEy$#yG{^1r1-+bW2-kF{JqyR1Wg=#Fh=-_lsI0V zveMoMzBu<|`+kb_p?r1CI}wgg|5JicwUO>al}Pw?Dk6fUPzv)<5d?xJmu3mBm#CEE zMRdtsVYzViX2`AUa5Gfk`Ld+5G+xJpJcl% z|0LUG5r|*xb585`?|S3E$9m-ubcvP`bO^zAF>@dk+l7`mgwS@esyW2Uv;+vH?bKH& z4{#DD*iMz&5b-(~GK#KIhFD2F46%}Uc!Y`v%_u{Nc+i*|VkPnL2o(?ID^<1%wxbVK zyo%Mp5F#FG=IBGjgRTa?4;2rpK86tS7}AG`hn|Ou2d&63Dpe8>6wckVy23bXH(MvB zE~aA{a`vaID)LrCSOcyV6%V;k4TzVCm5ChP1wFXxsSlic|K|?G`s{wQmZ%hTA4)r~ zyAgHtj^y`QyFLUr4vGp?-MMC^8-Eka%)fm$nTM{6Z}_WsJ#qApf0H4Ec3v-5SX~#F z#2!M|#Z=M#A*G6kVxq2#hx8%hq4p`dE>_&tb+Pi5u8XNY`{O>Pgj91pLV1_2Ook9j zIHV8ZUA24Cb+L+|u8URH>biJHA7W)v`z>7;D_@biIOvtA+}t(r3ge+GXa=p}r3l4;4XON%s^EmGr5{@RIYae_(NT z=8o=`!fPLpNgNHwQybgio{+r5mEN6oQ|@WU%1qu3&-UK+y7jA8g@5m?uhv?5tyyi| z*ZXm2!_jK9dbm30{?7GvTW7=IOncG0dz}rHR;y7x+-~K~?j7BBrI^eZ8Oq?83chf* zN!)eVjbnXt3eDsV9H864PH)Y&@kV!}*{V0@o87ze!D#QA!M_h1(p`7M07HR|XnkX& zQmx(DoUI>2525ijbJdyIR(IL0=VYP@ACrlm-t^tvIW=3UP37N>$KF6v+lpq7Sc{)6Pl%Ot{$2A%Z#ezu@7Yos!rv$2+Ctb*L{Oq( zK!3<;8xP~V?v&@*J3BU-DEY_t_eALg(<5(oPMxjXUujG?p;%5Nok-l(=@Z5&QM}}j z`3LPyBR4F}=C%4f{#WaER$qT>x3jTXovBriRHuBhc207Cl^@M(tvhS&`NI(7@<|Z9 zO0#p}LZg1TVsd%Yk(=?vQ+1)*sw{f!X;V)<7xKmwT3Gg#pub8hr_W4aGV%GOk8?SF zv{fl^`Ql}3Sov&b8ZV7Dfavi%ECJ*^z)Vj{S|!_hGwpd`5KeMyW%-Y&Sm*~*h5c;g zE%)x24?giN*`!#v$tGw>Y^axhSd+tO(aqji0r@smOa*I1?pLWlGRhh*Z(1b zTJLn|M(j>{UsPD%+i|S0Qp!q?&ivKye&e<2C7mjmfOyZZyze8EA0zb&ubZ;cueM&? zy5*s-GXzcbtn~PmzkS`iKlTxV;1ZveKJe9>4{ZKg^xn=8k;+Q%-+tl5-+t+$qz_e0 zVdn4p+S<4N&W9O7%;S;WZ`|_Y)9xh*HIIkw9*H67ij|dq`Dy;ZqSCRf^zdg&uRsiw zK2*I0#M8h3r?5KpA^KWAaF^~6^pZwtKZcQ(-RS|}Gy!Yvk?mai% ztpXw9@#ug2=kLDgmwql8l~O$P@r4cVxpCx83?b(6*k7FfBZp49Dn`fx4NQE;v7fp7 z<_|K2n8#y(_|>2P^{vUfnyv<;d&#Wy$e(Zd#vi=x`$->sVFcovnD+xW6cHwPbbkb2 zRsP`82Y>z`Lx_1ivg6M_@#x3jMG$Hp4}awEZ|IzNh#^Eg-dq0AgWKmnnjq5ssa2L) z>3wYs>KA^5A;di1d+cW)e&&{M5`>(`kFNUEht7X3L#TQDmwSdgI>tdf$d!Vxf)K$! z`i+@i{I~V~^}MMZ=livp1Gzu>w!$@9HEee~I7dmZ8QeLFwSD~M_kS+?z_}0LNUVQ~ zp5xj0`TDDC)z%(chE~#G=@T`ZUZveRyVa=FnhW(t>(&$WtbO>LMe(j^aJqz<#(}jH zkgQa^)*nD{N|1b{B7B&mcP=_Y-tpGJh5-a?k?N)R^uS_>>_rU>y03aE z-h2&u!6sUHtoXRh00LE3y%g_M!-+B$Z^&}g%a}V+jvoW;_u=#}0)#t?`020!F*#Gi zqWtmwRULzgY zO4c6gzaF&MSTsC%k7r;knHql^2IP?xE%zRl9<(dDS*|5sOGz0xR_?te+i%yc`J7y2 za+k-fb8 zgSY(apZw+d2}E)?D^0$q`9q)i>^BHR^m6Dc>k?zh;jHwEuvpuI+@~ke*S=i7|Jo0~ z^Qpg0e5Dzkl`3cb+~2hS<-g*AXnMH z=7UiiO(_$$FDnY|5+Ea__?xgW52C9AI$A21#jl z&S@RP&F6jB*XxBM&dyQKFaB`x+GqwFj#ZB4{aSU=-`$)ys&)6~rTyLM&Mt+Yz0j!6 zR~q--ax9;li<&ua@y}A|xy^j4UYi!+^NT-C;itI)*&xUJqddbs8Gma!qfOJ*Ca#E^ z_+Z9jLN+_=78-f8*>2?hCN3^qI!`w03%IDByQPIEW?YTjy9IA{o?9?H&~7x`jdor% zI&YzlXMkGO%G^yy#&*mv%<3PH5C8GD;XmG<{`fhI`W_z*eY@8N26I?!vTbRT?Ms{N zSlVQQn_Mu{s5F~%mD=?6__)b_e841c6oYkc!Q+$mEK>;Fa#qb1Us?8UycZ`fKYHsY zZo%|BmV@P=vh{UvFaUMD)|ak<$iOrj>$twicPkS;mx#Yf$iMw$n%EP%{&3ymMNU-S z{>VwI{2zRkuqX8e4Bf+*qXQdT==h*}UrSoNz{TF^)twO`Y=^=}5KR!K>KLRSkHH9F z8pUJqScQ}P=?WesyrmVMjVWfyg_YKPy@{u8phx?K?7d*Fe(W{dY($=SxDUlHS%iFUhv<$nV%+bHSugSncbabH|%cf%xs`W5@(lLcC+x$`i3QO-FOgJdQQ1J-Vrf7z}kL^H8|#)KWs zjcDvv>qS(_^hZHLSF86or{x}}|9z8bpRdRD^3sg;@umG{9Onm%_kK*^eDaf3uHn4< z`Q7trobNQheX1_TyI0b!!z4wO(Q$!7r<^ zJ~v=gWJXGxVB6zk#xvS~*d%3=Y6EtnDp4u%$ztv@2@|?umNPU-T3dnyc^kVJbtVb* LSR&v+lzsV zZ7563DNUW??x2|5LiT(TfU%a^c{fl4U-aUWua{A-%?*07d zPg;Z4U%kBl+skk6pa0`8)AKhU?!L57H*a;ay?u52_T%YGdw2To&BJ#$Z$8+S?{ED5 zt3TaMua2MTF3V@PU%j_$-QGOd`iHOHfB9~_x_LOhx_xn3V4>c- zSIX2Q8j#u)AQpls{^kpEs~KRxPA;J+J6WXFC|r=Gm5k-s|l(q0u@&%6FCT5M&!a~a!WCO1~yUFcF4*JS>?JAS6ejF zhPENaB&R2@=${zP+@1t1mX?)-w6GU4Tc{d%d~;rbM0FcbSF)4a302&|stYXAm=m~N zJHk@*8UisU6ti=}3NE8WE$A~Fv`kEZmMT#UGwC}8Y-LF73Fo*ZvUbA8GXpIIBaE9= zQdY_5(yS*J85|?bM4lk=RpNmv?uX+H_}mv=87}985R#2{N$QeA;bZdjl`-OmV(Fw4(E4bzoM&)pcdB$4fJyyS~o~d z{!mCS>PC``o$|5B{Ru50>T*)-e=N#B@zgmJO!LLSWm7<>G=LiC6HU?t#~oR#=~slV zatZtLhX@OmUz)ajj*~16i%*IdLYIkVB$ln97BnplQYZqSB88xGwLvBX7t9nl5~iJ* z`qk)VofM*`ES=D3pW%hXAE=_Cig;smL4wPOSwRMYl9n@@5|GCr0aO|TR|{CJ!g*5r z4}rG$JVectYOd@QV16W@L_-dC$T@e1jKRqrAa5>^hJWhpa7Y+CW^_nYP<{|dTZDybuv^~?&!!I{5qu|H z($W%7BG}23(eqqX*<$vMi~?Hbr$(C$iaSK}B~=-f&(1NaQOl%*A;ZFW?>)aGJLNK} zi=nA52363ld7UF8S&qmqX>IX)x*gp9Shr`|3j$QtAF>c27(?4M53*8l89Wk{%u?GR zJL%*paF7`_BJhl)&^cdxuxk)et>hneDY@8ui0FBl)jIIhml78`An6 zr0+@`W%%?@(bEd@>?8x{&>P8`13t~K&tW}kT&>^ed2bb~*W;^MeQO|mCVs=yt;%5A z9-h?8iY=5ERR>@#5>*1N&H-!p$c|W&w>-b$g2ShASznlUePLMqUX-I%7TGkwtnTrl zT*Is!rV`9>nh+yt$v@JjQJGLYwE_oG4)AblN<`Pbzd08d0Y8e%0=bQ2SS+85L;$g1 zt!oz*gEdo;dX25(>>Nc?A_b^Rs@Lkb5WAlWO+HB+$Z4}M%178)*i~5AGXgs!Z1H?9 zlX%)uj08;HiJ0V4Fr~yv@Ws&-2d@SYoe^VvjN+0gXEa8VJ0&0p*v=+&d+3+!n&@qMm>aygp38g&UgNxd`+_CS`>9&64(Rd_v?b6~LWgcA?LF>p3&gXog9c(QFP zF(3`e@108!H?2h2jFFb&iXiO`cCY%Jaa3a}(^8-*vR=xP(&?NfFnS&$$cbK6k{GJ7 z0S%bIxpjvvCOVRZC0aT`+H|O3T4jZP@lx#5XsVbXw<<$c))b$YAdJdN4#uE~iDnj- zR1(A{dBBHiD7fB}iD=>6bG4{StkX%5?WT6qRAcx6oaS9ky-`Q&AoxtMF?P)%84s}f z36NksKo@1}1m_A-kL3hWi<|^SnS_Va2TBbVls??O6NoVs^1Z%M0;S4-(j*4JFtwEp z-CQB>U14b&Pyn@=-p?dOrirPccYGZ*&;T;SqRdz0fY3Ek5AS5rfK4l5RZAwSX7Lol zE2u~gXpNgzh7?>|26_OMK(Ii(#4e>UiYmdhs9TVOs?U;jGUZz?GMA1myQ-Qfg<(~+ z(pC+Jzk-ZA2b+7Bj`Wm(YJMpZJ)cT*c(#s+0P(6hG5jmv6ENjq4pwKJd+MTj}Fr3h*!HDDU!6*2+S zpf9D@W1n2~9%~r$m`Drk7pfmpAhoUc1YNde+3KFI)!K2a5k`5S>|so<(Y8GpmK8?n zTf9M;1~CicaSsg2w3CFwD3emyVx|5BOojPE8100%9TW-U>hNS0PYhw?q9%Eo z;=z8P${-&xMrhwaja5fff&B*U<8qXyz&X^;R~HaT>S`Z7f)^fn1IHbz)!s2b>tj>~ zb<=^03==APs>J+7&#B-soZ6 zh^dOda9mD5MNxhBLYYC^#c=U=w(A1DB#W>ZVM;PHDJ{UoeQ+!~bJ8smRERo6{+7U~ zEkh?UG(p=Agvy~IBQ0$Yi5oLL#7ounwY}>kPHM2gtu4zmO)9U`mPS!E-W_|lUC+2X zGfo34;=rMmh6BHR0~ZePF}NY|-mcfNDHV|w&CGlA=KcA--`g+dzWFhwWzRSa2-ciI8e*TC^ z8uo*H`z2TBg~QRr+5<8qkMoZ~wc|54|8e#1wR%ANl*vntfOkk>CyMh!5-1v>YuQJN zhR03TeD;?VNSq<`xIga#&lW5weS&n~sxEtwZ4DLVliqm>ZXUFbGhmisZa7kfx8J_bN zUvza|`Z7Qmbo~G#EXOH}RH8k1V7Ct=?`^$0A&qe|Qh0RG=Ps?Zs=JN)6$93O*Qc)S zGly1~3W7t!XJ8e|UiqxK+pf26S9cCx5@XKXfPf2Q>2zd?^zaFKb9Uqn=CVatkEZ^v zLfJg-HeFG{l1#atic;M-;Gts9GIPbPwpA)y8|BTbdDB|Ihc9G93V;-VuT=fgwn^8m z?F}=xS+d}sMcv#s*>rQJxoK{eI@Beb=H%6;n_eB>3i;3%fEN-NE|J85b~A+PJfMy= zBsW5ui1S8+<9@OMQ2$@x_h>LEd^nge3c!ZsgHa#)2{8_D<ABrIO*_&{q(_-^9P8`3 zeY;^tis&=w-964uyjgW=m#9Eu-=H9I`BPZ!b-RMfNG|{@0H7j$7Ei044yHLyO{8aU z+E7`2!6nCm-}9tRhCb_wNn>6UEtJrLcLOdU6!b-mWnWNHd+}oQRO#g&NpV4LIwK4u3g3_s5?zKOBL1 zB%Z(qAMSnv*;L*|pp-bjaZ|$P3_1vS8_-Z5gUYDwPeZug43re4sCs=2&WyDM0Jf|R zl+Sz0IT4+iY#Sb3WCpQ2|Uo!2XD|QG@32NZ6xS3`cc*$LLZj4M?3KsKc2HWr_z@oLs$9 zg8?w4t`R#2fK}iq#Hus6NB}$EmFw`J(2G2NM4)^jS&!jAcasu^`OD(a@rCpqh%`|e znSnH!>i~QaS9qCmpnj6P+xRvA<>xrz_1QG4iF)fRy%euM+m|cit5b5({ zOw4W-EqM2lA_1~Cpc3#iQ;mqu0O|bSKvKzsm@Sn6Io|@)J}2IH6Z31Dy#XLN@kcKu zuIQEVK^(RBT)#R#nJUx=)d8{{q#lWGpgMt{%wHpyGaz{QJ`m8p24QZ@rS!;fcz+PU z40rP|I>%N4MK?~3!RJYxRm1yX|KWWHAWZc6SP=ltnGAWbLx(iLq%0(}W90U^WH;sE zDMXTBdH+b=$Xp@1J3H8F8}}l literal 0 HcmV?d00001 diff --git a/tests/test_unit_conversion.py b/tests/test_unit_conversion.py new file mode 100644 index 0000000..26091b3 --- /dev/null +++ b/tests/test_unit_conversion.py @@ -0,0 +1,636 @@ +""" +Tests for the optional pint unit conversion feature across every client. + +Approach (per the agreed plan): real API payloads are captured once by +``scratch/capture_unit_fixtures.py`` and committed as pickles under +``tests/data/unit_fixtures``. Each test loads a pickle, mocks the client's +network boundary to return it, and asserts two things: + + 1. Units are correctly *inferred* from the returned payload (the + ``{variable.name}_units`` column matches the source string). + 2. Passing ``desired_units`` converts the value column (as magnitudes) and + updates the units column to the requested unit. + +The raw payloads are fed through the real parsing code so unit inference stays +under test; only the network layer is mocked. +""" +import pickle +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock, patch + +import numpy as np +import pytest +import shapely + +from metloom.unit_conversions import ( + UREG, convert_series, normalize_unit, +) + +FIXTURE_DIR = Path(__file__).parent.joinpath("data/unit_fixtures") + + +def load_fixture(name): + with open(FIXTURE_DIR.joinpath(f"{name}.pkl"), "rb") as fp: + return pickle.load(fp) + + +# --------------------------------------------------------------------------- +# unit_conversions module +# --------------------------------------------------------------------------- +class TestUnitConversionsModule: + @pytest.mark.parametrize("raw, expected", [ + ("DEG F", "degF"), + ("deg C", "degC"), + ("INCHES", "inch"), + ("CFS", "foot**3/second"), + ("ac-ft", "acre_foot"), + ("%", "percent"), + ("wmoUnit:degC", "degC"), + ("Watts/meter^2", "watt/meter**2"), + ("Celsius", "degC"), + ("Millimeters", "millimeter"), + # already-valid / unknown strings pass through unchanged + ("degC", "degC"), + ("some_unknown_unit", "some_unknown_unit"), + ]) + def test_normalize_unit(self, raw, expected): + assert normalize_unit(raw) == expected + + def test_normalize_none(self): + assert normalize_unit(None) is None + + def test_convert_offset_units(self): + # 32 degF -> 0 degC, 212 degF -> 100 degC, NaN preserved + result = convert_series(np.array([32.0, 212.0, np.nan]), "DEG F", "degC") + assert result[0] == pytest.approx(0.0, abs=1e-6) + assert result[1] == pytest.approx(100.0, abs=1e-6) + assert np.isnan(result[2]) + + def test_convert_ratio_units(self): + result = convert_series(np.array([1.0, 2.0]), "INCHES", "mm") + np.testing.assert_allclose(result, [25.4, 50.8]) + + def test_incompatible_dimensions_returns_unchanged(self): + # meters -> degC is nonsense; should warn and return values unchanged + values = np.array([1.0, 2.0]) + result = convert_series(values, "meters", "degC") + np.testing.assert_array_equal(result, values) + + def test_unknown_unit_returns_unchanged(self): + values = np.array([1.0, 2.0]) + result = convert_series(values, "flibbers", "mm") + np.testing.assert_array_equal(result, values) + + def test_shared_registry(self): + # everything must build quantities on the same registry + assert convert_series.__globals__["UREG"] is UREG + + +# --------------------------------------------------------------------------- +# CDEC (units inferred from the API 'units' field: INCHES / DEG F) +# --------------------------------------------------------------------------- +class TestCDECUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import CDECPointData + pt = shapely.geometry.Point(-119.449875, 37.837581, 8150.0) + return CDECPointData("TNY", "Tenaya Lake", metadata=pt) + + @pytest.fixture + def mock_requests(self): + data = load_fixture("cdec") + + def side_effect(url, **kwargs): + mock = MagicMock() + params = kwargs.get("params") or {} + if params.get("SensorNums") == "3": + mock.json.return_value = data["swe"] + elif params.get("SensorNums") == "30": + mock.json.return_value = data["temp"] + else: + mock.json.return_value = [] + return mock + + with patch("metloom.pointdata.cdec.requests") as mock_requests: + mock_requests.get.side_effect = side_effect + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import CdecStationVariables + return station.get_daily_data( + datetime(2021, 5, 15), datetime(2021, 5, 18), + [CdecStationVariables.SWE, CdecStationVariables.TEMPAVG], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["SWE_units"].dropna().unique().tolist() == ["INCHES"] + assert df["AVG AIR TEMP_units"].dropna().unique().tolist() == ["DEG F"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units={"SWE": "mm", + "AVG AIR TEMP": "degC"}) + # units columns updated + assert conv["SWE_units"].unique().tolist() == ["mm"] + assert conv["AVG AIR TEMP_units"].unique().tolist() == ["degC"] + # values converted as magnitudes + raw_swe = raw["SWE"].to_numpy(dtype=float) + np.testing.assert_allclose( + conv["SWE"].to_numpy(dtype=float), raw_swe * 25.4, equal_nan=True + ) + raw_t = raw["AVG AIR TEMP"].to_numpy(dtype=float) + np.testing.assert_allclose( + conv["AVG AIR TEMP"].to_numpy(dtype=float), (raw_t - 32.0) * 5.0 / 9.0, + equal_nan=True, atol=1e-6 + ) + # not pint Quantities + assert conv["SWE"].to_numpy(dtype=float).dtype == float + + +# --------------------------------------------------------------------------- +# USGS (units inferred from unitCode: ft3/s) +# --------------------------------------------------------------------------- +class TestUSGSUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import USGSPointData + pt = shapely.geometry.Point(-106.54, 37.35, 9866.6) + st = USGSPointData("08245000", "Conejos", metadata=pt) + return st + + @pytest.fixture + def mock_response(self): + payload = load_fixture("usgs_daily") + with patch( + "metloom.pointdata.usgs.USGSPointData._get_url_response" + ) as mock_resp, patch( + "metloom.pointdata.usgs.USGSPointData._get_tzinfo" + ) as mock_tz: + mock_resp.return_value = payload + mock_tz.return_value = timezone(timedelta(hours=-7)) + yield mock_resp + + def _get(self, station, desired_units=None): + from metloom.variables import USGSVariables + return station.get_daily_data( + datetime(2020, 7, 1), datetime(2020, 7, 2), + [USGSVariables.DISCHARGE], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_response): + df = self._get(station) + assert df["DISCHARGE_units"].unique().tolist() == ["ft3/s"] + + def test_conversion(self, station, mock_response): + raw = self._get(station) + conv = self._get(station, desired_units="m^3/s") + assert conv["DISCHARGE_units"].unique().tolist() == ["m^3/s"] + expected = raw["DISCHARGE"].to_numpy(dtype=float) * 0.028316846592 + np.testing.assert_allclose( + conv["DISCHARGE"].to_numpy(dtype=float), expected, rtol=1e-6 + ) + + +# --------------------------------------------------------------------------- +# SNOTEL (units inferred from element storedUnitCd: in / degF) +# --------------------------------------------------------------------------- +class TestSnotelUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import SnotelPointData + pt = shapely.geometry.Point(-107.67552, 37.9339, 9800.0) + return SnotelPointData("538:CO:SNTL", "eh", metadata=pt) + + @pytest.fixture + def mocks(self): + fx = load_fixture("snotel_daily") + + def data_client_factory(*args, **kwargs): + client = MagicMock() + client.DURATION = "DAILY" + + def get_data(element_cd=None, **kw): + return fx["data"].get(element_cd, []) + client.get_data.side_effect = get_data + return client + + with patch( + "metloom.pointdata.snotel.DailySnotelDataClient", + side_effect=data_client_factory, + ), patch( + "metloom.pointdata.snotel.SnotelPointData._get_all_elements", + return_value=fx["elements"], + ), patch( + "metloom.pointdata.snotel.SnotelPointData._get_tzinfo", + return_value=timezone(timedelta(hours=fx["tz_hours"])), + ): + yield + + def _get(self, station, desired_units=None): + from metloom.variables import SnotelVariables + return station.get_daily_data( + datetime(2020, 3, 20), datetime(2020, 3, 22), + [SnotelVariables.SWE, SnotelVariables.TEMP], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mocks): + df = self._get(station) + assert df["SWE_units"].unique().tolist() == ["in"] + assert df["AIR TEMP_units"].unique().tolist() == ["degF"] + + def test_conversion(self, station, mocks): + raw = self._get(station) + conv = self._get(station, desired_units={"SWE": "mm", "AIR TEMP": "degC"}) + assert conv["SWE_units"].unique().tolist() == ["mm"] + assert conv["AIR TEMP_units"].unique().tolist() == ["degC"] + np.testing.assert_allclose( + conv["SWE"].to_numpy(dtype=float), + raw["SWE"].to_numpy(dtype=float) * 25.4, + ) + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + (raw["AIR TEMP"].to_numpy(dtype=float) - 32.0) * 5.0 / 9.0, + atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# Mesowest (units inferred from the 'UNITS' map: Celsius / Millimeters) +# --------------------------------------------------------------------------- +class TestMesowestUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import MesowestPointData + with patch( + "metloom.pointdata.mesowest.MesowestPointData.token", + new_callable=PropertyMock, return_value="faketoken", + ): + pt = shapely.geometry.Point(-119.5, 38.0, 7000) + yield MesowestPointData("INMTP", "test", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("mesowest") + resp = MagicMock() + resp.json.return_value = payload + with patch("metloom.pointdata.mesowest.requests.get", + return_value=resp) as mock_get: + yield mock_get + + def _get(self, station, desired_units=None): + from metloom.variables import MesowestVariables + return station.get_hourly_data( + datetime(2021, 3, 16), datetime(2021, 3, 16, 2), + [MesowestVariables.TEMP, MesowestVariables.SNOWDEPTH], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["Celsius"] + assert df["SNOWDEPTH_units"].unique().tolist() == ["Millimeters"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units={"AIR TEMP": "degF", + "SNOWDEPTH": "m"}) + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + assert conv["SNOWDEPTH_units"].unique().tolist() == ["m"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + np.testing.assert_allclose( + conv["SNOWDEPTH"].to_numpy(dtype=float), + raw["SNOWDEPTH"].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# GeoSphere Austria (units inferred from parameter 'unit': cm) +# --------------------------------------------------------------------------- +class TestGeoSphereUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import GeoSphereHistPointData + pt = shapely.geometry.Point(11.700833, 47.5075, 3074.14708) + return GeoSphereHistPointData("8807", "Tester", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("geosphere_hist") + resp = MagicMock() + resp.json.return_value = payload + with patch("metloom.pointdata.geosphere_austria.requests.get", + return_value=resp) as mock_get: + yield mock_get + + def _get(self, station, desired_units=None): + from metloom.variables import GeoSphereHistVariables + return station.get_daily_data( + datetime(2023, 1, 20), datetime(2023, 1, 25), + [GeoSphereHistVariables.SNOWDEPTH], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["Snowdepth_units"].unique().tolist() == ["cm"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="m") + assert conv["Snowdepth_units"].unique().tolist() == ["m"] + np.testing.assert_allclose( + conv["Snowdepth"].to_numpy(dtype=float), + raw["Snowdepth"].to_numpy(dtype=float) / 100.0, + ) + + +# --------------------------------------------------------------------------- +# CUES (units parsed out of the returned CSV column header: Watts/meter^2) +# --------------------------------------------------------------------------- +class TestCuesUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import CuesLevel1 + return CuesLevel1(None, None) + + @pytest.fixture + def mock_requests(self): + text = load_fixture("cues_daily") + resp = MagicMock() + resp.content = text.encode() + with patch("metloom.pointdata.cues.requests") as mock_requests: + mock_requests.post.return_value = resp + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import CuesLevel1Variables + return station.get_daily_data( + datetime(2020, 3, 15), datetime(2020, 3, 17), + [CuesLevel1Variables.DOWNSHORTWAVE], desired_units=desired_units, + ) + + @property + def col(self): + from metloom.variables import CuesLevel1Variables + return CuesLevel1Variables.DOWNSHORTWAVE.name + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df[f"{self.col}_units"].unique().tolist() == ["Watts/meter^2"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="kW/m^2") + assert conv[f"{self.col}_units"].unique().tolist() == ["kW/m^2"] + np.testing.assert_allclose( + conv[self.col].to_numpy(dtype=float), + raw[self.col].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# NWS forecast (units inferred from wmoUnit code: wmoUnit:degC) +# --------------------------------------------------------------------------- +class TestNWSUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import NWSForecastPointData + fx = load_fixture("nws") + + def side_effect(url, *args, **kwargs): + obj = MagicMock() + if "/gridpoints" in url: + obj.json.return_value = fx["data"] + else: + obj.json.return_value = fx["initial"] + return obj + + with patch("requests.get", side_effect=side_effect): + yield NWSForecastPointData( + "test", None, + initial_metadata=shapely.geometry.Point(-119, 43), + metadata=shapely.geometry.Point(-119, 43, 1000), + ) + + def _get(self, station, desired_units=None): + from metloom.variables import NWSForecastVariables + return station.get_hourly_forecast( + [NWSForecastVariables.TEMP], desired_units=desired_units, + ) + + def test_infer_units(self, station): + # NWS parsing strips the wmoUnit: namespace, leaving degC + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["degC"] + + def test_conversion(self, station): + raw = self._get(station) + conv = self._get(station, desired_units="degF") + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# MetNorway (units inferred from observation 'unit': degC) +# --------------------------------------------------------------------------- +class TestNorwayUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import MetNorwayPointData + pt = shapely.geometry.Point(8.0, 61.0, 500) + with patch( + "metloom.pointdata.norway.MetNorwayPointData.auth_header", + new_callable=PropertyMock, return_value={}, + ): + yield MetNorwayPointData("SN47610", "x", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("norway_hourly") + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = payload + with patch("metloom.pointdata.norway.requests") as mock_requests: + mock_requests.get.return_value = resp + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import MetNorwayVariables + return station.get_hourly_data( + datetime(2023, 8, 1), datetime(2023, 8, 2), + [MetNorwayVariables.TEMP], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["degC"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="degF") + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# SnowEx (static units from the SensorDescription: deg C / w/m^2) +# --------------------------------------------------------------------------- +class TestSnowExUnits: + @pytest.fixture + def station(self, tmp_path): + from metloom.pointdata import SnowExMet + return SnowExMet("LSOS", cache=str(tmp_path)) + + @pytest.fixture + def mock_download(self, tmp_path): + text = load_fixture("snowex_lsos") + + def _download(self, urls): + fp = tmp_path.joinpath("snowex.csv") + fp.write_text(text) + return [fp] + + with patch( + "metloom.pointdata.files.CSVPointData._download", _download + ): + yield + + @property + def temp_col(self): + from metloom.variables import SnowExVariables + return SnowExVariables.TEMP_20FT.name + + @property + def rad_col(self): + from metloom.variables import SnowExVariables + return SnowExVariables.UPSHORTWAVE.name + + def _get(self, station, desired_units=None): + from metloom.variables import SnowExVariables + return station.get_hourly_data( + datetime(2017, 1, 1), datetime(2017, 1, 1, 3), + [SnowExVariables.TEMP_20FT, SnowExVariables.UPSHORTWAVE], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_download): + df = self._get(station) + assert df[f"{self.temp_col}_units"].unique().tolist() == ["deg C"] + assert df[f"{self.rad_col}_units"].unique().tolist() == ["w/m^2"] + + def test_conversion(self, station, mock_download): + raw = self._get(station) + conv = self._get(station, desired_units={ + self.temp_col: "degF", + self.rad_col: "kW/m^2", + }) + assert conv[f"{self.temp_col}_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv[self.temp_col].to_numpy(dtype=float), + raw[self.temp_col].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, + atol=1e-6, + ) + np.testing.assert_allclose( + conv[self.rad_col].to_numpy(dtype=float), + raw[self.rad_col].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# CSAS (static units from the SensorDescription: meters / deg C) +# --------------------------------------------------------------------------- +class TestCSASUnits: + @pytest.fixture + def station(self, tmp_path): + from metloom.pointdata import CSASMet + return CSASMet("SBSP", cache=str(tmp_path)) + + @pytest.fixture + def mock_download(self, tmp_path): + text = load_fixture("csas_sbsp") + + def _download(self, urls): + fp = tmp_path.joinpath("csas.csv") + fp.write_text(text) + return [fp] + + with patch( + "metloom.pointdata.files.CSVPointData._download", _download + ): + yield + + def _get(self, station, desired_units=None): + from metloom.variables import CSASVariables + return station.get_hourly_data( + datetime(2023, 3, 1), datetime(2023, 3, 1, 3), + [CSASVariables.SNOWDEPTH, CSASVariables.SURF_TEMP], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_download): + df = self._get(station) + assert df["SNOWDEPTH_units"].unique().tolist() == ["meters"] + assert df["SURFACE TEMP_units"].unique().tolist() == ["deg C"] + + def test_conversion(self, station, mock_download): + raw = self._get(station) + conv = self._get(station, desired_units={"SNOWDEPTH": "inch", + "SURFACE TEMP": "degF"}) + assert conv["SNOWDEPTH_units"].unique().tolist() == ["inch"] + assert conv["SURFACE TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["SNOWDEPTH"].to_numpy(dtype=float), + raw["SNOWDEPTH"].to_numpy(dtype=float) / 0.0254, rtol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# SAIL/ARM (static units from the SensorDescription extra: w/m^2) +# --------------------------------------------------------------------------- +class TestSAILUnits: + @pytest.fixture + def station(self): + from metloom.pointdata.sail import SAILPointData + return SAILPointData("GUC:M1") + + @pytest.fixture + def mock_arm(self): + df = load_fixture("sail_precip") + with patch( + "metloom.pointdata.sail.arm_utils.get_station_data", + return_value=df, + ) as mock: + yield mock + + def _get(self, station, desired_units=None): + from metloom.variables import SAILStationVariables + return station.get_daily_data( + datetime(2023, 1, 1), datetime(2023, 1, 2), + [SAILStationVariables.PRECIPITATION], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_arm): + df = self._get(station) + assert df["PRECIPITATION_units"].unique().tolist() == ["mm"] + + def test_conversion(self, station, mock_arm): + raw = self._get(station) + conv = self._get(station, desired_units="inch") + col = "PRECIPITATION" + assert conv[f"{col}_units"].unique().tolist() == ["inch"] + np.testing.assert_allclose( + conv[col].to_numpy(dtype=float), + raw[col].to_numpy(dtype=float) / 25.4, rtol=1e-6, + )