diff --git a/metloom/pointdata/base.py b/metloom/pointdata/base.py index 47f0591..eefbc78 100644 --- a/metloom/pointdata/base.py +++ b/metloom/pointdata/base.py @@ -1,12 +1,13 @@ import copy import logging from datetime import datetime -from typing import List +from typing import Dict, List, Union import pandas as pd import geopandas as gpd +from ..unit_conversions import convert_series from ..variables import SensorDescription, VariableBase LOG = logging.getLogger("metloom.pointdata.base") @@ -168,6 +169,62 @@ def validate_sensor_df(cls, gdf: gpd.GeoDataFrame): if "_units" not in rc: assert f"{rc}_units" in remaining_columns + def _convert_units( + self, + gdf: gpd.GeoDataFrame, + desired_units: Union[str, Dict[str, str], None], + ): + """ + Optionally convert variable columns to a desired unit using pint. + + This is a no-op unless ``desired_units`` is provided - when it is + ``None`` pint is never invoked and the dataframe is returned unchanged. + + Args: + gdf: the assembled GeoDataFrame returned by a ``get_*`` method. + May be ``None`` or empty. + desired_units: either a single unit string applied to every + variable column, or a dict mapping ``variable.name`` to a unit + string. Units strings are pint-compatible (e.g. ``"degC"``, + ``"mm"``, ``"W/m^2"``). + Returns: + The GeoDataFrame with the requested variable columns converted in + place. Converted values are plain magnitudes (never pint + Quantities) and the matching ``{name}_units`` column is updated to + the target unit. NaNs are preserved. + """ + # Short-circuit before touching pint when no conversion was requested + if desired_units is None or gdf is None or len(gdf) == 0: + return gdf + + # variable columns are those that have a sibling `{col}_units` column + variable_columns = [ + c for c in gdf.columns if f"{c}_units" in gdf.columns + ] + for column in variable_columns: + if isinstance(desired_units, str): + target = desired_units + else: + target = desired_units.get(column) + if target is None: + continue + + units_column = f"{column}_units" + # infer the current unit from the first non-null entry + current = gdf[units_column].dropna() + if len(current) == 0: + LOG.warning( + "No units found for %s; skipping conversion", column + ) + continue + source_unit = current.iloc[0] + + gdf[column] = convert_series( + gdf[column].to_numpy(), source_unit, target + ) + gdf[units_column] = target + return gdf + def __repr__(self): return f"{self.__class__.__name__}({self.id!r}, {self.name!r})" @@ -197,6 +254,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get daily measurement data @@ -205,6 +263,10 @@ def get_daily_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns @@ -222,6 +284,7 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get hourly measurement data @@ -230,6 +293,10 @@ def get_hourly_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns @@ -247,6 +314,7 @@ def get_snow_course_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units: Union[str, Dict[str, str]] = None, ): """ Get snow course data @@ -255,6 +323,10 @@ def get_snow_course_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion. Either a + single unit string applied to every variable, or a dict + mapping ``variable.name`` to a unit string. When omitted no + conversion is performed and the inferred units are returned. Returns: GeoDataFrame of data. The dataframe should be indexed on ['datetime', 'site'] and have columns diff --git a/metloom/pointdata/cdec.py b/metloom/pointdata/cdec.py index 838e301..6ae5222 100644 --- a/metloom/pointdata/cdec.py +++ b/metloom/pointdata/cdec.py @@ -228,7 +228,8 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], duration_list: List[str], - include_measurement_date=False + include_measurement_date=False, + desired_units=None, ): """ Args: @@ -240,6 +241,8 @@ def _get_data( include_measurement_date: boolean for including the 'measurmentDate' column in the resulting dataframe. This column is only relevant for snow courses + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -280,6 +283,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_event_data( @@ -287,14 +291,18 @@ def get_event_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): - return self._get_data(start_date, end_date, variables, ["E"]) + return self._get_data( + start_date, end_date, variables, ["E"], desired_units=desired_units + ) def get_daily_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -302,24 +310,32 @@ def get_daily_data( https://cdec.water.ca.gov/dynamicapp/req/JSONDataServlet? Stations=TNY&SensorNums=3&dur_code=D&Start=2021-05-16&End=2021-05-16 """ - return self._get_data(start_date, end_date, variables, ["D", "h", "E"]) + return self._get_data( + start_date, end_date, variables, ["D", "h", "E"], + desired_units=desired_units, + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ - return self._get_data(start_date, end_date, variables, ["h", "E"]) + return self._get_data( + start_date, end_date, variables, ["h", "E"], + desired_units=desired_units, + ) def get_snow_course_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_snow_course_data @@ -327,7 +343,8 @@ def get_snow_course_data( if not self.is_partly_snow_course(): raise ValueError(f"{self.id} is not a snow course") return self._get_data( - start_date, end_date, variables, ["M"], include_measurement_date=True + start_date, end_date, variables, ["M"], + include_measurement_date=True, desired_units=desired_units, ) @staticmethod diff --git a/metloom/pointdata/cues.py b/metloom/pointdata/cues.py index bf89cdb..9c7b76a 100644 --- a/metloom/pointdata/cues.py +++ b/metloom/pointdata/cues.py @@ -114,7 +114,7 @@ def _sensor_response_to_df(self, data, variable): def _get_data( self, start_date, end_date, variables: List[SensorDescription], - period, + period, desired_units=None, ): df = pd.DataFrame() df.index.name = "datetime" @@ -133,18 +133,23 @@ def _get_data( df = gpd.GeoDataFrame(df, geometry=[self.metadata] * len(df)) df = df.reset_index().set_index(["datetime", "site"]) self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "day" + start_date, end_date, variables, "day", + desired_units=desired_units, ) def get_hourly_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "hr" + start_date, end_date, variables, "hr", + desired_units=desired_units, ) def get_snow_course_data(self, start_date: datetime, end_date: datetime, diff --git a/metloom/pointdata/files.py b/metloom/pointdata/files.py index 7e445b3..54887b9 100644 --- a/metloom/pointdata/files.py +++ b/metloom/pointdata/files.py @@ -169,7 +169,7 @@ def _get_one_variable(self, resp_df, period, variable: SensorDescription): return data def _get_data(self, start_date, end_date, variables: List[SensorDescription], - period): + period, desired_units=None): """ Utilizes cached data or downloads the data """ @@ -222,18 +222,23 @@ def _get_data(self, start_date, end_date, variables: List[SensorDescription], df = df.reset_index().set_index(["datetime", "site"]) self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "D" + start_date, end_date, variables, "D", + desired_units=desired_units, ) def get_hourly_data(self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription]): + variables: List[SensorDescription], + desired_units=None): return self._get_data( - start_date, end_date, variables, "h" + start_date, end_date, variables, "h", + desired_units=desired_units, ) def _get_metadata(self): diff --git a/metloom/pointdata/geosphere_austria.py b/metloom/pointdata/geosphere_austria.py index cdaa754..4dd7f0f 100644 --- a/metloom/pointdata/geosphere_austria.py +++ b/metloom/pointdata/geosphere_austria.py @@ -172,6 +172,7 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], desired_duration: str, + desired_units=None, ): """ @@ -181,6 +182,8 @@ def _get_data( variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES desired_duration: duration code ['D', 'h'] + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -212,6 +215,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df @classmethod @@ -346,6 +350,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -354,19 +359,24 @@ def get_daily_data( tawes-v1-10min?parameters=TL&station_ids=11035 """ self._validate_dates(end_date) - return self._get_data(start_date, end_date, variables, "D") + return self._get_data( + start_date, end_date, variables, "D", desired_units=desired_units + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ self._validate_dates(end_date) - return self._get_data(start_date, end_date, variables, "h") + return self._get_data( + start_date, end_date, variables, "h", desired_units=desired_units + ) class GeoSphereHistPointData(GeoSpherePointDataBase): @@ -406,6 +416,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -413,4 +424,6 @@ def get_daily_data( https://dataset.api.hub.geosphere.at/v1/station/historical/klima-v1-1d ?station_ids=11401&start=2023-04-12&end=2023-04-14¶meters=schnee """ - return self._get_data(start_date, end_date, variables, None) + return self._get_data( + start_date, end_date, variables, None, desired_units=desired_units + ) diff --git a/metloom/pointdata/mesowest.py b/metloom/pointdata/mesowest.py index cd51427..bea0244 100644 --- a/metloom/pointdata/mesowest.py +++ b/metloom/pointdata/mesowest.py @@ -91,7 +91,8 @@ def _get_data(self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], - interval='H'): + interval='H', + desired_units=None): """ Make get request to Mesowest and return JSON Args: @@ -100,6 +101,8 @@ def _get_data(self, variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES interval: String interval the resulting data is resampled to + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: dictionary of response values """ @@ -136,6 +139,7 @@ def _get_data(self, else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df @staticmethod @@ -244,6 +248,7 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get hourly measurement data @@ -262,7 +267,10 @@ def get_hourly_data( TestCDECStation.tny_daily_expected for example dataframe. Datetimes should be in UTC """ - df = self._get_data(start_date, end_date, variables, interval='H') + df = self._get_data( + start_date, end_date, variables, interval='H', + desired_units=desired_units, + ) return df def get_daily_data( @@ -270,6 +278,7 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get daily measurement data @@ -288,7 +297,10 @@ def get_daily_data( TestCDECStation.tny_daily_expected for example dataframe. Datetimes should be in UTC """ - df = self._get_data(start_date, end_date, variables, interval='D') + df = self._get_data( + start_date, end_date, variables, interval='D', + desired_units=desired_units, + ) return df @classmethod diff --git a/metloom/pointdata/norway.py b/metloom/pointdata/norway.py index bdbbe7c..3d9c1dc 100644 --- a/metloom/pointdata/norway.py +++ b/metloom/pointdata/norway.py @@ -360,6 +360,7 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], desired_duration=None, + desired_units=None, ): """ Args: @@ -367,6 +368,8 @@ def _get_data( end_date: datetime object for end of data collection period variables: List of metloom.variables.SensorDescription object from self.ALLOWED_VARIABLES + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -396,6 +399,7 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data( @@ -403,12 +407,14 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data """ return self._get_data( - start_date, end_date, variables, desired_duration="D" + start_date, end_date, variables, desired_duration="D", + desired_units=desired_units, ) def get_hourly_data( @@ -416,12 +422,14 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data """ return self._get_data( - start_date, end_date, variables, desired_duration="h" + start_date, end_date, variables, desired_duration="h", + desired_units=desired_units, ) def get_event_data( @@ -429,12 +437,14 @@ def get_event_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ Get data in original frequency from API """ return self._get_data( - start_date, end_date, variables, desired_duration=None + start_date, end_date, variables, desired_duration=None, + desired_units=desired_units, ) @classmethod diff --git a/metloom/pointdata/nws_forecast.py b/metloom/pointdata/nws_forecast.py index 5ebb64b..0d8e479 100644 --- a/metloom/pointdata/nws_forecast.py +++ b/metloom/pointdata/nws_forecast.py @@ -212,6 +212,7 @@ def _get_data( self, variables: List[SensorDescription], desired_duration=None, + desired_units=None, ): """ Args: @@ -219,6 +220,8 @@ def _get_data( from self.ALLOWED_VARIABLES desired_duration: desired resample duration ("D", "h"). Data is hourly be default + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -247,11 +250,13 @@ def _get_data( else: df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with daily results for a 7 day forecast. @@ -260,11 +265,14 @@ def get_daily_forecast( Args: variables: list of variables to return """ - return self._get_data(variables, desired_duration="D") + return self._get_data( + variables, desired_duration="D", desired_units=desired_units + ) def get_hourly_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with hourly results for a 7 day forecast. @@ -273,11 +281,14 @@ def get_hourly_forecast( Args: variables: list of variables to return """ - return self._get_data(variables, desired_duration="h") + return self._get_data( + variables, desired_duration="h", desired_units=desired_units + ) def get_forecast( self, variables: List[SensorDescription], + desired_units=None, ): """ Get a geopandas dataframe with hourly results for a 7 day forecast. @@ -287,4 +298,4 @@ def get_forecast( variables: list of variables to return """ # Do not resample - return self._get_data(variables) + return self._get_data(variables, desired_units=desired_units) diff --git a/metloom/pointdata/sail.py b/metloom/pointdata/sail.py index cac219c..06ff9e4 100644 --- a/metloom/pointdata/sail.py +++ b/metloom/pointdata/sail.py @@ -52,18 +52,26 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: list[SensorDescription], + desired_units=None, ): self._check_start_end_dates(start_date, end_date) - return self._download_sail_data(start_date, end_date, variables, interval="D") + return self._download_sail_data( + start_date, end_date, variables, interval="D", + desired_units=desired_units, + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, variables: list[SensorDescription], + desired_units=None, ): self._check_start_end_dates(start_date, end_date) - return self._download_sail_data(start_date, end_date, variables, interval="h") + return self._download_sail_data( + start_date, end_date, variables, interval="h", + desired_units=desired_units, + ) def _download_sail_data( self, @@ -71,6 +79,7 @@ def _download_sail_data( end_date: datetime, variables: list[SensorDescription], interval: str, + desired_units=None, ) -> pd.DataFrame: """ The ARM data is stored in a series of files based on the sensors at the location. @@ -121,6 +130,7 @@ def _download_sail_data( df["datasource"] = "ARM" df.reset_index(inplace=True) df = df.set_index(["datetime", "site"]) + df = self._convert_units(df, desired_units) return df else: LOG.error( diff --git a/metloom/pointdata/snotel.py b/metloom/pointdata/snotel.py index f978d92..b0709c6 100644 --- a/metloom/pointdata/snotel.py +++ b/metloom/pointdata/snotel.py @@ -112,6 +112,7 @@ def _fetch_data_for_variables(self, client: SeriesSnotelClient, variables: List[SensorDescription], duration: str, include_measurement_date=False, + desired_units=None, ): result_map = {} for variable in variables: @@ -121,15 +122,17 @@ def _fetch_data_for_variables(self, client: SeriesSnotelClient, result_map[variable] = data else: LOG.warning(f"No {variable.name} found for {self.name}") - return self._snotel_response_to_df( + df = self._snotel_response_to_df( result_map, duration, include_measurement_date=include_measurement_date ) + return self._convert_units(df, desired_units) def get_daily_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data @@ -139,14 +142,16 @@ def get_daily_data( begin_date=start_date, end_date=end_date, ) - return self._fetch_data_for_variables(client, variables, - client.DURATION) + return self._fetch_data_for_variables( + client, variables, client.DURATION, desired_units=desired_units + ) def get_hourly_data( self, start_date: datetime, end_date: datetime, - variables: List[SensorDescription] + variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_hourly_data @@ -157,13 +162,16 @@ def get_hourly_data( begin_date=start_date, end_date=end_date, ) - return self._fetch_data_for_variables(client, variables, "HOURLY") + return self._fetch_data_for_variables( + client, variables, "HOURLY", desired_units=desired_units + ) def get_snow_course_data( self, start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_snow_course_data @@ -174,7 +182,8 @@ def get_snow_course_data( end_date=end_date, ) return self._fetch_data_for_variables( - client, variables, client.DURATION, include_measurement_date=True + client, variables, client.DURATION, + include_measurement_date=True, desired_units=desired_units, ) def _get_all_metadata(self): diff --git a/metloom/pointdata/usgs.py b/metloom/pointdata/usgs.py index 12183be..8740bb2 100644 --- a/metloom/pointdata/usgs.py +++ b/metloom/pointdata/usgs.py @@ -269,7 +269,8 @@ def _get_data( end_date: datetime, variables: List[SensorDescription], duration_list: List[str], - resample_duration=None + resample_duration=None, + desired_units=None, ): """ Args: @@ -279,6 +280,8 @@ def _get_data( from self.ALLOWED_VARIABLES duration_list: list of USGS duration code, "dv" or "iv" resample_duration: optional if we need to resample the data + desired_units: Optional pint-compatible unit conversion (see + PointData.get_daily_data) Returns: GeoDataFrame of data, indexed on datetime, site """ @@ -329,6 +332,7 @@ def _get_data( df = None self.validate_sensor_df(df) + df = self._convert_units(df, desired_units) return df def get_daily_data( @@ -336,13 +340,14 @@ def get_daily_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ See docstring for PointData.get_daily_data """ return self._get_data( start_date, end_date, variables, ["dv", "iv"], - resample_duration="24H" + resample_duration="24H", desired_units=desired_units, ) def get_hourly_data( @@ -350,9 +355,11 @@ def get_hourly_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): return self._get_data( - start_date, end_date, variables, ["iv"], resample_duration="h" + start_date, end_date, variables, ["iv"], resample_duration="h", + desired_units=desired_units, ) def get_instantaneous_data( @@ -360,11 +367,15 @@ def get_instantaneous_data( start_date: datetime, end_date: datetime, variables: List[SensorDescription], + desired_units=None, ): """ USGS 'instantaneous' data, which is generally 15 minutes. """ - return self._get_data(start_date, end_date, variables, ["iv"]) + return self._get_data( + start_date, end_date, variables, ["iv"], + desired_units=desired_units, + ) @staticmethod def _get_url_response(url, params=None, parse='text'): diff --git a/metloom/unit_conversions.py b/metloom/unit_conversions.py new file mode 100644 index 0000000..11b19f9 --- /dev/null +++ b/metloom/unit_conversions.py @@ -0,0 +1,133 @@ +""" +Optional unit conversion helpers built on ``pint``. + +metloom clients infer a units string for every variable they return (stored in +the ``{variable.name}_units`` column). Those strings come from a variety of +external APIs and are not always directly parseable by pint +(e.g. ``"DEG F"``, ``"w/m^2"``, ``"CFS"``, ``"wmoUnit:degC"``). This module +normalizes those strings and provides a single conversion entry point. + +pint is only exercised when a caller explicitly requests a conversion by +passing ``desired_units`` to a ``get_*`` method. When no conversion is +requested none of this code runs and the returned data is unchanged. +""" +import logging + +import numpy as np +import pint + +LOG = logging.getLogger("metloom.unit_conversions") + +# A single shared registry. pint Quantities can only interact when they share +# the same UnitRegistry, so everything in metloom must use this instance. +UREG = pint.UnitRegistry() + +# Map the messy unit strings returned by the various data sources onto strings +# that ``UREG`` can parse. Keys are compared case-insensitively after stripping +# surrounding whitespace (see ``normalize_unit``). +_NORMALIZE = { + # temperature + "deg f": "degF", + "degf": "degF", + "deg_f": "degF", + "°f": "degF", + "deg c": "degC", + "degc": "degC", + "deg_c": "degC", + "°c": "degC", + "celsius": "degC", + "fahrenheit": "degF", + # length / depth + "inches": "inch", + "in": "inch", + "feet": "foot", + "ft": "foot", + "meters": "meter", + "metres": "meter", + "millimeters": "millimeter", + "millimetres": "millimeter", + "centimeters": "centimeter", + "centimetres": "centimeter", + # irradiance + "w/m^2": "watt/meter**2", + "w/m2": "watt/meter**2", + "w m-2": "watt/meter**2", + "watt/m^2": "watt/meter**2", + "watt/meter^2": "watt/meter**2", + "watts/meter^2": "watt/meter**2", + "watts per square meter": "watt/meter**2", + # flow / volume + "cfs": "foot**3/second", + "ft3/s": "foot**3/second", + "ft^3/s": "foot**3/second", + "cubic feet per second": "foot**3/second", + "ac-ft": "acre_foot", + "acre-ft": "acre_foot", + "ac_ft": "acre_foot", + # ratios / angles + "pct": "percent", + "%": "percent", + "degrees": "degree", + # rates + "in/hr": "inch/hour", + "m/s": "meter/second", +} + + +def normalize_unit(raw): + """ + Normalize a source units string into something pint can parse. + + Args: + raw: the units string inferred by a client (e.g. ``"DEG F"``, + ``"wmoUnit:degC"``). May be ``None``. + Returns: + A pint-parseable string, or ``None`` if ``raw`` is ``None``. Unknown + strings are returned unchanged so anything already valid still works. + """ + if raw is None: + return None + key = str(raw).strip() + if not key: + return None + # NWS returns units like "wmoUnit:degC" - strip the namespace prefix + if ":" in key and key.lower().startswith("wmounit"): + key = key.split(":", 1)[1] + return _NORMALIZE.get(key.lower(), key) + + +def convert_series(values, from_unit, to_unit): + """ + Convert an array of values from one unit to another. + + Offset units (degF <-> degC) are handled correctly via ``pint.Quantity``. + NaNs are preserved. On any failure (undefined unit, incompatible + dimensions) a warning is logged and the values are returned unchanged so a + unit hiccup never crashes a data pull. + + Args: + values: array-like of numeric values + from_unit: source units string (raw, will be normalized) + to_unit: target units string (raw, will be normalized) + Returns: + numpy array of converted magnitudes (plain floats, never a + pint.Quantity), or the original values if conversion was not possible. + """ + from_norm = normalize_unit(from_unit) + to_norm = normalize_unit(to_unit) + if from_norm is None or to_norm is None: + LOG.warning( + "Cannot convert with missing units (from=%r, to=%r); " + "returning values unchanged", from_unit, to_unit + ) + return values + try: + magnitude = np.asarray(values, dtype="float64") + quantity = UREG.Quantity(magnitude, from_norm) + return quantity.to(to_norm).magnitude + except Exception as e: + LOG.warning( + "Failed converting from %r to %r (%s); returning values unchanged", + from_unit, to_unit, e + ) + return values diff --git a/setup.py b/setup.py index d30a058..ea5144a 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ 'beautifulsoup4>4,<5', 'zeep>4.0.0', 'pydash>=8.0.0,<9.0.0', + 'pint>=0.20,<1.0', ] test_requirements = ['pytest>=3', ] diff --git a/tests/capture_unit_fixtures.py b/tests/capture_unit_fixtures.py new file mode 100644 index 0000000..1bd589f --- /dev/null +++ b/tests/capture_unit_fixtures.py @@ -0,0 +1,232 @@ +""" +Dev-only helper: capture the raw payloads each metloom client parses and +pickle them under ``tests/data/unit_fixtures/`` so the unit-conversion tests +can run fully offline against real data shapes. + +This is NOT collected by pytest (no ``test_`` prefix). Run it once to +(re)generate fixtures:: + + ./venv/bin/python tests/capture_unit_fixtures.py + +Strategy per the agreed plan: + * Pull genuinely live data where it is cheap and credential-free (CDEC). + * For everything else, source the payloads from the committed real-response + mock files already in ``tests/data/*_mocks`` (these are captured real API + responses) and re-serialize them as pickles. Payloads whose canonical + form is not stored as a file (Mesowest / SNOTEL / SAIL) are reproduced + here in the exact shape the live APIs return. + +Every fixture is the *raw* payload (parsed JSON / CSV text / row lists), NOT a +finished DataFrame, so the client parsing + unit inference stays under test. +""" +import json +import pickle +from pathlib import Path + +HERE = Path(__file__).resolve().parent +MOCKS = HERE / "data" +OUT = MOCKS / "unit_fixtures" + + +def _dump(name, obj): + OUT.mkdir(parents=True, exist_ok=True) + with open(OUT / f"{name}.pkl", "wb") as fp: + pickle.dump(obj, fp) + print(f"wrote {name}.pkl") + + +def capture_cdec(): + """Live pull of real CDEC data for Tenaya Lake (TNY).""" + try: + import requests + url = "http://cdec.water.ca.gov/dynamicapp/req/JSONDataServlet" + out = {} + for key, sensor in [("swe", "3"), ("temp", "30")]: + resp = requests.get(url, params={ + "Stations": "TNY", "dur_code": "D", "SensorNums": sensor, + "Start": "2021-05-15T00:00:00", "End": "2021-05-18T00:00:00", + }, timeout=20) + resp.raise_for_status() + out[key] = resp.json() + # sanity: make sure we got units in the payload + assert out["swe"] and "units" in out["swe"][0] + _dump("cdec", out) + except Exception as e: # pragma: no cover - network dependent + print(f"CDEC live pull failed ({e}); writing static fallback") + out = { + "swe": [ + {"stationId": "TNY", "durCode": "D", "SENSOR_NUM": 3, + "date": f"2021-5-{d} 00:00", "obsDate": f"2021-5-{d} 00:00", + "value": v, "dataFlag": " ", "units": "INCHES"} + for d, v in [(16, 12.1), (17, 12.0), (18, 11.8)] + ], + "temp": [ + {"stationId": "TNY", "durCode": "D", "SENSOR_NUM": 30, + "date": f"2021-5-{d} 00:00", "obsDate": f"2021-5-{d} 00:00", + "value": v, "dataFlag": " ", "units": "DEG F"} + for d, v in [(16, 33.1), (17, 34.2), (18, 35.0)] + ], + } + _dump("cdec", out) + + +def capture_usgs(): + with open(MOCKS / "usgs_mocks" / "daily_response.txt") as fp: + _dump("usgs_daily", json.load(fp)) + + +def capture_geosphere(): + """ + GeoSphere data responses are GeoJSON FeatureCollections with a top-level + ``timestamps`` list and per-parameter ``unit``/``data`` under + ``features[0].properties.parameters``. (klima_mock.json is the *metadata* + endpoint, not the data endpoint, so it is not used here.) + """ + payload = { + "media_type": "application/json", "type": "FeatureCollection", + "version": "v1", + "timestamps": [ + "2023-01-20T00:00+00:00", "2023-01-21T00:00+00:00", + "2023-01-22T00:00+00:00", "2023-01-23T00:00+00:00", + "2023-01-24T00:00+00:00", "2023-01-25T00:00+00:00", + ], + "features": [{ + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [11.700833, 47.5075]}, + "properties": { + "parameters": { + "schnee": { + "name": "Gesamtschneehoehe", + "unit": "cm", + "data": [3.0, 18.0, 22.0, 18.0, 18.0, 14.0], + } + }, + "station": "8807", + }, + }], + } + _dump("geosphere_hist", payload) + + +def capture_cues(): + with open(MOCKS / "cues_mocks" / "daily_response.txt") as fp: + _dump("cues_daily", fp.read()) + + +def capture_nws(): + with open(MOCKS / "nws_mocks" / "meta_and_data.json") as fp: + meta_and_data = json.load(fp) + with open(MOCKS / "nws_mocks" / "initial_meta.json") as fp: + initial_meta = json.load(fp) + _dump("nws", {"initial": initial_meta, "data": meta_and_data}) + + +def capture_norway(): + with open(MOCKS / "frost_mocks" / "hourly_temp.json") as fp: + _dump("norway_hourly", json.load(fp)) + + +def _csv_subset(path, n_rows=48): + """Header + first n_rows data lines of a csv, as text.""" + with open(path) as fp: + lines = fp.readlines() + return "".join(lines[: n_rows + 1]) + + +def capture_snowex(): + _dump( + "snowex_lsos", + _csv_subset(MOCKS / "snowex_mocks" / "SNEX_Met_LSOS_final_output.csv"), + ) + + +def capture_csas(): + _dump( + "csas_sbsp", + _csv_subset(MOCKS / "csas_mocks" / "SBSP_1hr_2010-2024.csv"), + ) + + +def capture_mesowest(): + """ + Reproduce the Mesowest timeseries response shape (units=metric). Real + responses nest observations under STATION[0]['OBSERVATIONS'] and report + the per-variable units under 'UNITS'. + """ + dts = [ + "2021-03-16T00:00:00Z", "2021-03-16T01:00:00Z", "2021-03-16T02:00:00Z", + ] + payload = { + "SUMMARY": {"RESPONSE_MESSAGE": "OK"}, + "UNITS": {"air_temp": "Celsius", "snow_depth": "Millimeters"}, + "STATION": [{ + "STID": "INMTP", + "OBSERVATIONS": { + "date_time": dts, + "air_temp_set_1": [-2.1, -1.4, -0.8], + "snow_depth_set_1": [1200.0, 1210.0, 1205.0], + }, + }], + } + _dump("mesowest", payload) + + +def capture_snotel(): + """ + SNOTEL data comes back from the AWDB SOAP service as lists of + {'datetime','value'} rows; units are inferred from the station elements + (storedUnitCd). Reproduce both. + """ + payload = { + "tz_hours": -8.0, + "elements": [ + {"elementCd": "WTEQ", "duration": "DAILY", "storedUnitCd": "in"}, + {"elementCd": "TOBS", "duration": "DAILY", "storedUnitCd": "degF"}, + ], + "data": { + "WTEQ": [ + {"datetime": "2020-03-20", "value": 13.19}, + {"datetime": "2020-03-21", "value": 13.17}, + {"datetime": "2020-03-22", "value": 13.14}, + ], + "TOBS": [ + {"datetime": "2020-03-20", "value": 30.2}, + {"datetime": "2020-03-21", "value": 31.5}, + {"datetime": "2020-03-22", "value": 33.8}, + ], + }, + } + _dump("snotel_daily", payload) + + +def capture_sail(): + """ + SAIL/ARM data is delivered as netCDF; arm_utils.get_station_data returns a + DataFrame indexed on datetime with a column per variable code. Reproduce a + small frame for the precipitation variable (mm). + """ + import pandas as pd + idx = pd.date_range("2023-01-01", periods=6, freq="h", name="datetime") + df = pd.DataFrame( + {"accum_rtnrt": [0.0, 0.1, 0.3, 0.0, 0.2, 0.5]}, index=idx + ) + _dump("sail_precip", df) + + +def main(): + capture_cdec() + capture_usgs() + capture_geosphere() + capture_cues() + capture_nws() + capture_norway() + capture_snowex() + capture_csas() + capture_mesowest() + capture_snotel() + capture_sail() + print(f"\nFixtures written to {OUT}") + + +if __name__ == "__main__": + main() diff --git a/tests/data/unit_fixtures/cdec.pkl b/tests/data/unit_fixtures/cdec.pkl new file mode 100644 index 0000000..6095c62 Binary files /dev/null and b/tests/data/unit_fixtures/cdec.pkl differ diff --git a/tests/data/unit_fixtures/csas_sbsp.pkl b/tests/data/unit_fixtures/csas_sbsp.pkl new file mode 100644 index 0000000..48d8172 Binary files /dev/null and b/tests/data/unit_fixtures/csas_sbsp.pkl differ diff --git a/tests/data/unit_fixtures/cues_daily.pkl b/tests/data/unit_fixtures/cues_daily.pkl new file mode 100644 index 0000000..45b93cc Binary files /dev/null and b/tests/data/unit_fixtures/cues_daily.pkl differ diff --git a/tests/data/unit_fixtures/geosphere_hist.pkl b/tests/data/unit_fixtures/geosphere_hist.pkl new file mode 100644 index 0000000..a2f4e27 Binary files /dev/null and b/tests/data/unit_fixtures/geosphere_hist.pkl differ diff --git a/tests/data/unit_fixtures/mesowest.pkl b/tests/data/unit_fixtures/mesowest.pkl new file mode 100644 index 0000000..f3e7c1d Binary files /dev/null and b/tests/data/unit_fixtures/mesowest.pkl differ diff --git a/tests/data/unit_fixtures/norway_hourly.pkl b/tests/data/unit_fixtures/norway_hourly.pkl new file mode 100644 index 0000000..b360aad Binary files /dev/null and b/tests/data/unit_fixtures/norway_hourly.pkl differ diff --git a/tests/data/unit_fixtures/nws.pkl b/tests/data/unit_fixtures/nws.pkl new file mode 100644 index 0000000..df1ee2b Binary files /dev/null and b/tests/data/unit_fixtures/nws.pkl differ diff --git a/tests/data/unit_fixtures/sail_precip.pkl b/tests/data/unit_fixtures/sail_precip.pkl new file mode 100644 index 0000000..65b3aa2 Binary files /dev/null and b/tests/data/unit_fixtures/sail_precip.pkl differ diff --git a/tests/data/unit_fixtures/snotel_daily.pkl b/tests/data/unit_fixtures/snotel_daily.pkl new file mode 100644 index 0000000..6fbd666 Binary files /dev/null and b/tests/data/unit_fixtures/snotel_daily.pkl differ diff --git a/tests/data/unit_fixtures/snowex_lsos.pkl b/tests/data/unit_fixtures/snowex_lsos.pkl new file mode 100644 index 0000000..2090b88 Binary files /dev/null and b/tests/data/unit_fixtures/snowex_lsos.pkl differ diff --git a/tests/data/unit_fixtures/usgs_daily.pkl b/tests/data/unit_fixtures/usgs_daily.pkl new file mode 100644 index 0000000..84d0ec2 Binary files /dev/null and b/tests/data/unit_fixtures/usgs_daily.pkl differ diff --git a/tests/test_unit_conversion.py b/tests/test_unit_conversion.py new file mode 100644 index 0000000..26091b3 --- /dev/null +++ b/tests/test_unit_conversion.py @@ -0,0 +1,636 @@ +""" +Tests for the optional pint unit conversion feature across every client. + +Approach (per the agreed plan): real API payloads are captured once by +``scratch/capture_unit_fixtures.py`` and committed as pickles under +``tests/data/unit_fixtures``. Each test loads a pickle, mocks the client's +network boundary to return it, and asserts two things: + + 1. Units are correctly *inferred* from the returned payload (the + ``{variable.name}_units`` column matches the source string). + 2. Passing ``desired_units`` converts the value column (as magnitudes) and + updates the units column to the requested unit. + +The raw payloads are fed through the real parsing code so unit inference stays +under test; only the network layer is mocked. +""" +import pickle +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock, patch + +import numpy as np +import pytest +import shapely + +from metloom.unit_conversions import ( + UREG, convert_series, normalize_unit, +) + +FIXTURE_DIR = Path(__file__).parent.joinpath("data/unit_fixtures") + + +def load_fixture(name): + with open(FIXTURE_DIR.joinpath(f"{name}.pkl"), "rb") as fp: + return pickle.load(fp) + + +# --------------------------------------------------------------------------- +# unit_conversions module +# --------------------------------------------------------------------------- +class TestUnitConversionsModule: + @pytest.mark.parametrize("raw, expected", [ + ("DEG F", "degF"), + ("deg C", "degC"), + ("INCHES", "inch"), + ("CFS", "foot**3/second"), + ("ac-ft", "acre_foot"), + ("%", "percent"), + ("wmoUnit:degC", "degC"), + ("Watts/meter^2", "watt/meter**2"), + ("Celsius", "degC"), + ("Millimeters", "millimeter"), + # already-valid / unknown strings pass through unchanged + ("degC", "degC"), + ("some_unknown_unit", "some_unknown_unit"), + ]) + def test_normalize_unit(self, raw, expected): + assert normalize_unit(raw) == expected + + def test_normalize_none(self): + assert normalize_unit(None) is None + + def test_convert_offset_units(self): + # 32 degF -> 0 degC, 212 degF -> 100 degC, NaN preserved + result = convert_series(np.array([32.0, 212.0, np.nan]), "DEG F", "degC") + assert result[0] == pytest.approx(0.0, abs=1e-6) + assert result[1] == pytest.approx(100.0, abs=1e-6) + assert np.isnan(result[2]) + + def test_convert_ratio_units(self): + result = convert_series(np.array([1.0, 2.0]), "INCHES", "mm") + np.testing.assert_allclose(result, [25.4, 50.8]) + + def test_incompatible_dimensions_returns_unchanged(self): + # meters -> degC is nonsense; should warn and return values unchanged + values = np.array([1.0, 2.0]) + result = convert_series(values, "meters", "degC") + np.testing.assert_array_equal(result, values) + + def test_unknown_unit_returns_unchanged(self): + values = np.array([1.0, 2.0]) + result = convert_series(values, "flibbers", "mm") + np.testing.assert_array_equal(result, values) + + def test_shared_registry(self): + # everything must build quantities on the same registry + assert convert_series.__globals__["UREG"] is UREG + + +# --------------------------------------------------------------------------- +# CDEC (units inferred from the API 'units' field: INCHES / DEG F) +# --------------------------------------------------------------------------- +class TestCDECUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import CDECPointData + pt = shapely.geometry.Point(-119.449875, 37.837581, 8150.0) + return CDECPointData("TNY", "Tenaya Lake", metadata=pt) + + @pytest.fixture + def mock_requests(self): + data = load_fixture("cdec") + + def side_effect(url, **kwargs): + mock = MagicMock() + params = kwargs.get("params") or {} + if params.get("SensorNums") == "3": + mock.json.return_value = data["swe"] + elif params.get("SensorNums") == "30": + mock.json.return_value = data["temp"] + else: + mock.json.return_value = [] + return mock + + with patch("metloom.pointdata.cdec.requests") as mock_requests: + mock_requests.get.side_effect = side_effect + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import CdecStationVariables + return station.get_daily_data( + datetime(2021, 5, 15), datetime(2021, 5, 18), + [CdecStationVariables.SWE, CdecStationVariables.TEMPAVG], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["SWE_units"].dropna().unique().tolist() == ["INCHES"] + assert df["AVG AIR TEMP_units"].dropna().unique().tolist() == ["DEG F"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units={"SWE": "mm", + "AVG AIR TEMP": "degC"}) + # units columns updated + assert conv["SWE_units"].unique().tolist() == ["mm"] + assert conv["AVG AIR TEMP_units"].unique().tolist() == ["degC"] + # values converted as magnitudes + raw_swe = raw["SWE"].to_numpy(dtype=float) + np.testing.assert_allclose( + conv["SWE"].to_numpy(dtype=float), raw_swe * 25.4, equal_nan=True + ) + raw_t = raw["AVG AIR TEMP"].to_numpy(dtype=float) + np.testing.assert_allclose( + conv["AVG AIR TEMP"].to_numpy(dtype=float), (raw_t - 32.0) * 5.0 / 9.0, + equal_nan=True, atol=1e-6 + ) + # not pint Quantities + assert conv["SWE"].to_numpy(dtype=float).dtype == float + + +# --------------------------------------------------------------------------- +# USGS (units inferred from unitCode: ft3/s) +# --------------------------------------------------------------------------- +class TestUSGSUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import USGSPointData + pt = shapely.geometry.Point(-106.54, 37.35, 9866.6) + st = USGSPointData("08245000", "Conejos", metadata=pt) + return st + + @pytest.fixture + def mock_response(self): + payload = load_fixture("usgs_daily") + with patch( + "metloom.pointdata.usgs.USGSPointData._get_url_response" + ) as mock_resp, patch( + "metloom.pointdata.usgs.USGSPointData._get_tzinfo" + ) as mock_tz: + mock_resp.return_value = payload + mock_tz.return_value = timezone(timedelta(hours=-7)) + yield mock_resp + + def _get(self, station, desired_units=None): + from metloom.variables import USGSVariables + return station.get_daily_data( + datetime(2020, 7, 1), datetime(2020, 7, 2), + [USGSVariables.DISCHARGE], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_response): + df = self._get(station) + assert df["DISCHARGE_units"].unique().tolist() == ["ft3/s"] + + def test_conversion(self, station, mock_response): + raw = self._get(station) + conv = self._get(station, desired_units="m^3/s") + assert conv["DISCHARGE_units"].unique().tolist() == ["m^3/s"] + expected = raw["DISCHARGE"].to_numpy(dtype=float) * 0.028316846592 + np.testing.assert_allclose( + conv["DISCHARGE"].to_numpy(dtype=float), expected, rtol=1e-6 + ) + + +# --------------------------------------------------------------------------- +# SNOTEL (units inferred from element storedUnitCd: in / degF) +# --------------------------------------------------------------------------- +class TestSnotelUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import SnotelPointData + pt = shapely.geometry.Point(-107.67552, 37.9339, 9800.0) + return SnotelPointData("538:CO:SNTL", "eh", metadata=pt) + + @pytest.fixture + def mocks(self): + fx = load_fixture("snotel_daily") + + def data_client_factory(*args, **kwargs): + client = MagicMock() + client.DURATION = "DAILY" + + def get_data(element_cd=None, **kw): + return fx["data"].get(element_cd, []) + client.get_data.side_effect = get_data + return client + + with patch( + "metloom.pointdata.snotel.DailySnotelDataClient", + side_effect=data_client_factory, + ), patch( + "metloom.pointdata.snotel.SnotelPointData._get_all_elements", + return_value=fx["elements"], + ), patch( + "metloom.pointdata.snotel.SnotelPointData._get_tzinfo", + return_value=timezone(timedelta(hours=fx["tz_hours"])), + ): + yield + + def _get(self, station, desired_units=None): + from metloom.variables import SnotelVariables + return station.get_daily_data( + datetime(2020, 3, 20), datetime(2020, 3, 22), + [SnotelVariables.SWE, SnotelVariables.TEMP], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mocks): + df = self._get(station) + assert df["SWE_units"].unique().tolist() == ["in"] + assert df["AIR TEMP_units"].unique().tolist() == ["degF"] + + def test_conversion(self, station, mocks): + raw = self._get(station) + conv = self._get(station, desired_units={"SWE": "mm", "AIR TEMP": "degC"}) + assert conv["SWE_units"].unique().tolist() == ["mm"] + assert conv["AIR TEMP_units"].unique().tolist() == ["degC"] + np.testing.assert_allclose( + conv["SWE"].to_numpy(dtype=float), + raw["SWE"].to_numpy(dtype=float) * 25.4, + ) + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + (raw["AIR TEMP"].to_numpy(dtype=float) - 32.0) * 5.0 / 9.0, + atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# Mesowest (units inferred from the 'UNITS' map: Celsius / Millimeters) +# --------------------------------------------------------------------------- +class TestMesowestUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import MesowestPointData + with patch( + "metloom.pointdata.mesowest.MesowestPointData.token", + new_callable=PropertyMock, return_value="faketoken", + ): + pt = shapely.geometry.Point(-119.5, 38.0, 7000) + yield MesowestPointData("INMTP", "test", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("mesowest") + resp = MagicMock() + resp.json.return_value = payload + with patch("metloom.pointdata.mesowest.requests.get", + return_value=resp) as mock_get: + yield mock_get + + def _get(self, station, desired_units=None): + from metloom.variables import MesowestVariables + return station.get_hourly_data( + datetime(2021, 3, 16), datetime(2021, 3, 16, 2), + [MesowestVariables.TEMP, MesowestVariables.SNOWDEPTH], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["Celsius"] + assert df["SNOWDEPTH_units"].unique().tolist() == ["Millimeters"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units={"AIR TEMP": "degF", + "SNOWDEPTH": "m"}) + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + assert conv["SNOWDEPTH_units"].unique().tolist() == ["m"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + np.testing.assert_allclose( + conv["SNOWDEPTH"].to_numpy(dtype=float), + raw["SNOWDEPTH"].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# GeoSphere Austria (units inferred from parameter 'unit': cm) +# --------------------------------------------------------------------------- +class TestGeoSphereUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import GeoSphereHistPointData + pt = shapely.geometry.Point(11.700833, 47.5075, 3074.14708) + return GeoSphereHistPointData("8807", "Tester", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("geosphere_hist") + resp = MagicMock() + resp.json.return_value = payload + with patch("metloom.pointdata.geosphere_austria.requests.get", + return_value=resp) as mock_get: + yield mock_get + + def _get(self, station, desired_units=None): + from metloom.variables import GeoSphereHistVariables + return station.get_daily_data( + datetime(2023, 1, 20), datetime(2023, 1, 25), + [GeoSphereHistVariables.SNOWDEPTH], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["Snowdepth_units"].unique().tolist() == ["cm"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="m") + assert conv["Snowdepth_units"].unique().tolist() == ["m"] + np.testing.assert_allclose( + conv["Snowdepth"].to_numpy(dtype=float), + raw["Snowdepth"].to_numpy(dtype=float) / 100.0, + ) + + +# --------------------------------------------------------------------------- +# CUES (units parsed out of the returned CSV column header: Watts/meter^2) +# --------------------------------------------------------------------------- +class TestCuesUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import CuesLevel1 + return CuesLevel1(None, None) + + @pytest.fixture + def mock_requests(self): + text = load_fixture("cues_daily") + resp = MagicMock() + resp.content = text.encode() + with patch("metloom.pointdata.cues.requests") as mock_requests: + mock_requests.post.return_value = resp + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import CuesLevel1Variables + return station.get_daily_data( + datetime(2020, 3, 15), datetime(2020, 3, 17), + [CuesLevel1Variables.DOWNSHORTWAVE], desired_units=desired_units, + ) + + @property + def col(self): + from metloom.variables import CuesLevel1Variables + return CuesLevel1Variables.DOWNSHORTWAVE.name + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df[f"{self.col}_units"].unique().tolist() == ["Watts/meter^2"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="kW/m^2") + assert conv[f"{self.col}_units"].unique().tolist() == ["kW/m^2"] + np.testing.assert_allclose( + conv[self.col].to_numpy(dtype=float), + raw[self.col].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# NWS forecast (units inferred from wmoUnit code: wmoUnit:degC) +# --------------------------------------------------------------------------- +class TestNWSUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import NWSForecastPointData + fx = load_fixture("nws") + + def side_effect(url, *args, **kwargs): + obj = MagicMock() + if "/gridpoints" in url: + obj.json.return_value = fx["data"] + else: + obj.json.return_value = fx["initial"] + return obj + + with patch("requests.get", side_effect=side_effect): + yield NWSForecastPointData( + "test", None, + initial_metadata=shapely.geometry.Point(-119, 43), + metadata=shapely.geometry.Point(-119, 43, 1000), + ) + + def _get(self, station, desired_units=None): + from metloom.variables import NWSForecastVariables + return station.get_hourly_forecast( + [NWSForecastVariables.TEMP], desired_units=desired_units, + ) + + def test_infer_units(self, station): + # NWS parsing strips the wmoUnit: namespace, leaving degC + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["degC"] + + def test_conversion(self, station): + raw = self._get(station) + conv = self._get(station, desired_units="degF") + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# MetNorway (units inferred from observation 'unit': degC) +# --------------------------------------------------------------------------- +class TestNorwayUnits: + @pytest.fixture + def station(self): + from metloom.pointdata import MetNorwayPointData + pt = shapely.geometry.Point(8.0, 61.0, 500) + with patch( + "metloom.pointdata.norway.MetNorwayPointData.auth_header", + new_callable=PropertyMock, return_value={}, + ): + yield MetNorwayPointData("SN47610", "x", metadata=pt) + + @pytest.fixture + def mock_requests(self): + payload = load_fixture("norway_hourly") + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = payload + with patch("metloom.pointdata.norway.requests") as mock_requests: + mock_requests.get.return_value = resp + yield mock_requests + + def _get(self, station, desired_units=None): + from metloom.variables import MetNorwayVariables + return station.get_hourly_data( + datetime(2023, 8, 1), datetime(2023, 8, 2), + [MetNorwayVariables.TEMP], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_requests): + df = self._get(station) + assert df["AIR TEMP_units"].unique().tolist() == ["degC"] + + def test_conversion(self, station, mock_requests): + raw = self._get(station) + conv = self._get(station, desired_units="degF") + assert conv["AIR TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["AIR TEMP"].to_numpy(dtype=float), + raw["AIR TEMP"].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, atol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# SnowEx (static units from the SensorDescription: deg C / w/m^2) +# --------------------------------------------------------------------------- +class TestSnowExUnits: + @pytest.fixture + def station(self, tmp_path): + from metloom.pointdata import SnowExMet + return SnowExMet("LSOS", cache=str(tmp_path)) + + @pytest.fixture + def mock_download(self, tmp_path): + text = load_fixture("snowex_lsos") + + def _download(self, urls): + fp = tmp_path.joinpath("snowex.csv") + fp.write_text(text) + return [fp] + + with patch( + "metloom.pointdata.files.CSVPointData._download", _download + ): + yield + + @property + def temp_col(self): + from metloom.variables import SnowExVariables + return SnowExVariables.TEMP_20FT.name + + @property + def rad_col(self): + from metloom.variables import SnowExVariables + return SnowExVariables.UPSHORTWAVE.name + + def _get(self, station, desired_units=None): + from metloom.variables import SnowExVariables + return station.get_hourly_data( + datetime(2017, 1, 1), datetime(2017, 1, 1, 3), + [SnowExVariables.TEMP_20FT, SnowExVariables.UPSHORTWAVE], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_download): + df = self._get(station) + assert df[f"{self.temp_col}_units"].unique().tolist() == ["deg C"] + assert df[f"{self.rad_col}_units"].unique().tolist() == ["w/m^2"] + + def test_conversion(self, station, mock_download): + raw = self._get(station) + conv = self._get(station, desired_units={ + self.temp_col: "degF", + self.rad_col: "kW/m^2", + }) + assert conv[f"{self.temp_col}_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv[self.temp_col].to_numpy(dtype=float), + raw[self.temp_col].to_numpy(dtype=float) * 9.0 / 5.0 + 32.0, + atol=1e-6, + ) + np.testing.assert_allclose( + conv[self.rad_col].to_numpy(dtype=float), + raw[self.rad_col].to_numpy(dtype=float) / 1000.0, + ) + + +# --------------------------------------------------------------------------- +# CSAS (static units from the SensorDescription: meters / deg C) +# --------------------------------------------------------------------------- +class TestCSASUnits: + @pytest.fixture + def station(self, tmp_path): + from metloom.pointdata import CSASMet + return CSASMet("SBSP", cache=str(tmp_path)) + + @pytest.fixture + def mock_download(self, tmp_path): + text = load_fixture("csas_sbsp") + + def _download(self, urls): + fp = tmp_path.joinpath("csas.csv") + fp.write_text(text) + return [fp] + + with patch( + "metloom.pointdata.files.CSVPointData._download", _download + ): + yield + + def _get(self, station, desired_units=None): + from metloom.variables import CSASVariables + return station.get_hourly_data( + datetime(2023, 3, 1), datetime(2023, 3, 1, 3), + [CSASVariables.SNOWDEPTH, CSASVariables.SURF_TEMP], + desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_download): + df = self._get(station) + assert df["SNOWDEPTH_units"].unique().tolist() == ["meters"] + assert df["SURFACE TEMP_units"].unique().tolist() == ["deg C"] + + def test_conversion(self, station, mock_download): + raw = self._get(station) + conv = self._get(station, desired_units={"SNOWDEPTH": "inch", + "SURFACE TEMP": "degF"}) + assert conv["SNOWDEPTH_units"].unique().tolist() == ["inch"] + assert conv["SURFACE TEMP_units"].unique().tolist() == ["degF"] + np.testing.assert_allclose( + conv["SNOWDEPTH"].to_numpy(dtype=float), + raw["SNOWDEPTH"].to_numpy(dtype=float) / 0.0254, rtol=1e-6, + ) + + +# --------------------------------------------------------------------------- +# SAIL/ARM (static units from the SensorDescription extra: w/m^2) +# --------------------------------------------------------------------------- +class TestSAILUnits: + @pytest.fixture + def station(self): + from metloom.pointdata.sail import SAILPointData + return SAILPointData("GUC:M1") + + @pytest.fixture + def mock_arm(self): + df = load_fixture("sail_precip") + with patch( + "metloom.pointdata.sail.arm_utils.get_station_data", + return_value=df, + ) as mock: + yield mock + + def _get(self, station, desired_units=None): + from metloom.variables import SAILStationVariables + return station.get_daily_data( + datetime(2023, 1, 1), datetime(2023, 1, 2), + [SAILStationVariables.PRECIPITATION], desired_units=desired_units, + ) + + def test_infer_units(self, station, mock_arm): + df = self._get(station) + assert df["PRECIPITATION_units"].unique().tolist() == ["mm"] + + def test_conversion(self, station, mock_arm): + raw = self._get(station) + conv = self._get(station, desired_units="inch") + col = "PRECIPITATION" + assert conv[f"{col}_units"].unique().tolist() == ["inch"] + np.testing.assert_allclose( + conv[col].to_numpy(dtype=float), + raw[col].to_numpy(dtype=float) / 25.4, rtol=1e-6, + )