From 71085bd6e5a331da81ca2d912d02611e396023ab Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:01:07 -0800 Subject: [PATCH 01/54] wip: draft new dataset module --- pyproject.toml | 1 + src/geodata/datasets/_base.py | 253 ++++++++++++++++++ src/geodata/datasets/{era5.py => _era5.py} | 0 src/geodata/datasets/era5/_base.py | 38 +++ .../datasets/era5/hourly/wind_solar.py | 119 ++++++++ src/geodata/datasets/hrrr/_base.py | 39 +++ src/geodata/datasets/hrrr/wind.py | 35 +++ src/geodata/datasets/hrrr/wind_solar.py | 82 ++++++ uv.lock | 102 +++++++ 9 files changed, 669 insertions(+) create mode 100644 src/geodata/datasets/_base.py rename src/geodata/datasets/{era5.py => _era5.py} (100%) create mode 100644 src/geodata/datasets/era5/_base.py create mode 100644 src/geodata/datasets/era5/hourly/wind_solar.py create mode 100644 src/geodata/datasets/hrrr/_base.py create mode 100644 src/geodata/datasets/hrrr/wind.py create mode 100644 src/geodata/datasets/hrrr/wind_solar.py diff --git a/pyproject.toml b/pyproject.toml index 9173310d..195652c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ license = {text = "GPLv3"} [project.optional-dependencies] download = [ "cdsapi>=0.7.3", + "herbie-data>=2024.8.0", ] notebook = [ "notebook>=7.2.2", diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py new file mode 100644 index 00000000..7cb02cf9 --- /dev/null +++ b/src/geodata/datasets/_base.py @@ -0,0 +1,253 @@ +# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import itertools +import logging +from collections.abc import Sequence +from pathlib import Path + +import pandas as pd +from tqdm.auto import tqdm + +from ..config import DATASET_ROOT_PATH + +logger = logging.getLogger(__name__) + + +class BaseDataset(abc.ABC): + """Dataset is a class that encapsulates any datasets natively supported + by geodata. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + + Args: + years: A slice object or sequence of two integers representing the + range of years to download. + months: A slice object or sequence of two integers representing the + range of months to download. + bounds (optional): A tuple of four floats representing the bounding box + (lon_min, lat_min, lon_max, lat_max) + **kwargs: Additional keyword arguments that are passed to the dataset. + + Notes: + - Subclasses of BaseDataset must define the following attributes: + - module: The module of the dataset. + - weather_config: The configuration of the dataset. + - Subclasses of BaseDataset must also implement the following methods: + - download: Method to download the dataset. + - _extra_setup: Method to handle any extra setup that is required + for the dataset. + - By default, the files downloaded by the dataset are defined to by monthly in + nature. That is, Subclasses can override this behavior by setting the `frequency` + + """ + + module: str + weather_config: str + frequency: str = "monthly" + + def __init__( + self, + years: Sequence[int] | slice, + months: Sequence[int] | slice, + bounds: Sequence[int] | None = None, + **kwargs, + ): + if not hasattr(self, "module"): + raise ValueError("Subclasses of BaseDataset must define a module attribute") + + if not hasattr(self, "weather_config"): + raise ValueError( + "Subclasses of BaseDataset must define a weather_config attribute" + ) + + if not isinstance(years, slice): + if isinstance(years, Sequence): + if not all(isinstance(year, int) for year in years): + raise ValueError("years must be a sequence of integers") + elif not len(years) == 2: + raise ValueError("years must be a sequence of length 2") + years = slice(years[0], years[1]) + else: + raise ValueError( + f"""Invalid input {years} for years. Years must either be a + sequence of integers or a slice object""" + ) + self.years = years + + if not isinstance(months, slice): + if isinstance(months, Sequence): + if not all(isinstance(month, int) for month in months): + raise ValueError("months must be a sequence of integers") + elif not len(months) == 2: + raise ValueError("months must be a sequence of length 2") + elif not all(1 <= month <= 12 for month in months): + raise ValueError( + "months must be a sequence of integers between 1 and 12" + ) + months = slice(months[0], months[1]) + else: + raise ValueError( + f"""Invalid input {months} for months. Months must either be a + sequence of integers or a slice object""" + ) + self.months = months + + if bounds is not None: + if not all(isinstance(bound, (int, float)) for bound in bounds): + raise ValueError("bounds must be a sequence of integers or floats") + if not len(bounds) == 4: + raise ValueError("bounds must be a sequence of length 4") + if not all(-180 <= bound <= 180 for bound in [bounds[0], bounds[2]]): + raise ValueError("Longitude bounds must be between -180 and 180") + if not all(-90 <= bound <= 90 for bound in [bounds[1], bounds[3]]): + raise ValueError("Latitude bounds must be between -90 and 90") + self.bounds = bounds + + self.storage_root = ( + Path(kwargs.get("dataset_root", DATASET_ROOT_PATH)) + / self.module + / self.weather_config + ) + if not self.storage_root.exists(): + logger.info( + f"""Storage directory for {self.__class__.__name__} + does not exist, creating now at {self.storage_root}""" + ) + self.storage_root.mkdir(parents=True) + + self._extra_setup(**kwargs) + + @abc.abstractmethod + def _extra_setup(self, **kwargs): + """Method to be implemented by subclasses to handle any extra setup + that is required for the dataset. + """ + + @property + @abc.abstractmethod + def downloaded(self): + """A boolean flag indicating whether the dataset has been prepared + for use. This typically means that the dataset has been downloaded, + preprocessed, and stored in a format that is ready for use. + """ + + @abc.abstractmethod + def _download_file(self, file: dict): + """Method to download a single file from the dataset. This method + should download the file and save it to the appropriate location. + + Args: + file: A dictionary containing the metadata of the file to download. At the + minimum, this dictionary should contain the following keys: + - year: the year of the file + - month: the month of the file + - day: the day of the file (if applicable) + - hour: the hour of the file (if applicable) + - save_path: the path where the file should be saved + """ + + def download(self): + """Method to download the dataset. This method should download the + dataset files and store them in the appropriate location. + """ + + for file in tqdm( + self.catalog, desc="Downloading", unit="file", dynamic_ncols=True + ): + self._download_file(file) + + def __repr__(self): + return "".format( + self.module, + self.weather_config, + self.years.start, + self.years.stop, + self.months.start, + self.months.stop, + "Prepared" if self.prepared else "Unprepared", + " " + self.extra_repr if self.extra_repr else "", + ) + + @property + def extra_repr(self): + return "" + + @property + def submodule(self): + """The submodule of the dataset. This can be defined by the dataset + using the `weather_config` attribute. If not defined, it will default + to the name of the dataset class. + """ + return getattr(self, "weather_config", self.__class__.__name__) + + @property + def catalog(self): + """A generator that yields all the files that need to be downloaded. + Each iteration should return a dictionary with the following keys: + - year: the year of the file + - month: the month of the file + - day: the day of the file (if applicable) + - hour: the hour of the file (if applicable) + - save_path: the path where the file should be saved + """ + + match self.frequency: + case "monthly": + yield from self._monthly_catalog() + case "daily": + yield from self._daily_catalog() + case "hourly": + yield from self._hourly_catalog() + case _: + raise ValueError( + f"Invalid frequency {self.frequency} defined for this dataset." + ) + + def _monthly_catalog(self): + for year, month in itertools.product( + range(self.years.start, self.years.stop + 1), + range(self.months.start, self.months.stop + 1), + ): + save_path = self.storage_root / f"{year}_{month:02d}.nc" + yield {"year": year, "month": month, "save_path": save_path} + + def _daily_catalog(self): + for year, month in itertools.product( + range(self.years.start, self.years.stop + 1), + range(self.months.start, self.months.stop + 1), + ): + for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): + save_path = self.storage_root / f"{year}_{month:02d}_{day:02d}.nc" + yield {"year": year, "month": month, "day": day, "save_path": save_path} + + def _hourly_catalog(self): + for year, month in itertools.product( + range(self.years.start, self.years.stop + 1), + range(self.months.start, self.months.stop + 1), + ): + for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): + for hour in range(24): + save_path = ( + self.storage_root + / f"{year}_{month:02d}_{day:02d}_{hour:02d}.nc" + ) + yield { + "year": year, + "month": month, + "day": day, + "hour": hour, + "save_path": save_path, + } diff --git a/src/geodata/datasets/era5.py b/src/geodata/datasets/_era5.py similarity index 100% rename from src/geodata/datasets/era5.py rename to src/geodata/datasets/_era5.py diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py new file mode 100644 index 00000000..41b20a4a --- /dev/null +++ b/src/geodata/datasets/era5/_base.py @@ -0,0 +1,38 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +import cdsapi + +from .._base import BaseDataset + + +class ERA5BaseDataset(BaseDataset): + """ERA5BaseDataset is a class that encaps a dataset from the ERA5 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + module = "era5" + + def _extra_setup(self, **kwargs): + self.logger = logging.getLogger(__name__.replace("._base", ".client")) + self.client = cdsapi.Client( + info_callback=self.logger.info, + error_callback=self.logger.error, + debug_callback=self.logger.debug, + warning_callback=self.logger.warning, + ) diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py new file mode 100644 index 00000000..be14caee --- /dev/null +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -0,0 +1,119 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import tempfile +import zipfile +from pathlib import Path + +import xarray as xr + +from .._base import ERA5BaseDataset + +logger = logging.getLogger(__name__) + + +class ERA5WindSolarHourlyDataset(ERA5BaseDataset): + """ERA5WindSolarHourlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind and solar + information. This dataset is stored in hourly intervals. + + The ERA5 dataset is a reanalysis dataset that provides a comprehensive + record of the Earth's climate. It is produced by the European Centre for + Medium-Range Weather Forecasts (ECMWF) and is available from 1980 to + present. + + Note: + - The specific variables that are downloaded are: + - 100m_u_component_of_wind + - 100m_v_component_of_wind + - 2m_temperature + - runoff + - soil_temperature_level_4 + - surface_net_solar_radiation + - surface_pressure + - surface_solar_radiation_downwards + - toa_incident_solar_radiation + - total_sky_direct_solar_radiation_at_surface + - forecast_surface_roughness + - geopotential + """ + + weather_config = "wind_solar_hourly" + + # Information that are needed for ERA5's API request + variables = { + "100m_u_component_of_wind": "u100", + "100m_v_component_of_wind": "v100", + "2m_temperature": "t2m", + "runoff": "ro", + "soil_temperature_level_4": "stl4", + "surface_net_solar_radiation": "ssr", + "surface_pressure": "sp", + "surface_solar_radiation_downwards": "ssrd", + "toa_incident_solar_radiation": "tisr", + "total_sky_direct_solar_radiation_at_surface": "fdir", + "forecast_surface_roughness": "fsr", + "geopotential": "z", + } + product = "reanalysis-era5-single-levels" + product_type = "reanalysis" + + def _download_file(self, file: dict): + year: int = file["year"] + month: int = file["month"] + save_path: Path = file["save_path"] + + full_request = { + "product_type": self.product_type, + "format": "netcdf", + "variable": list(self.variables.keys()), + "year": year, + "month": month, + "day": [f"{d:02d}" for d in range(1, 32)], + "time": [f"{t:02d}:00" for t in range(0, 24)], + } + + if self.bounds is not None: + full_request["area"] = self.bounds[::-1] + + full_result = self.client.retrieve(self.product, full_request) + if full_result.content_type == "application/zip": + logger.info( + "Multiple files found with request. Additional unzipping/preprocessing needed." + ) + + with tempfile.TemporaryDirectory() as tempdir: + full_result.download(os.path.join(tempdir, "download.zip")) + with zipfile.ZipFile( + os.path.join(tempdir, "download.zip"), "r" + ) as zip_ref: + zip_ref.extractall(tempdir) + + with xr.open_mfdataset( + [ + os.path.join(tempdir, f) + for f in os.listdir(tempdir) + if f.endswith(".nc") + ] + ) as ds: + ds.to_netcdf(save_path) + + logger.info("Preprocessing complete with zipfile") + logger.info("Successfully downloaded to %s", save_path) + + def downloaded(self): + pass diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py new file mode 100644 index 00000000..cd5f8cd2 --- /dev/null +++ b/src/geodata/datasets/hrrr/_base.py @@ -0,0 +1,39 @@ +# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import logging +import tempfile + + +from .._base import BaseDataset + +logger = logging.getLogger(__name__) + + +class HRRRBaseDataset(BaseDataset): + """HRRRBaseDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + module = "hrrr" + _priority = ["google", "aws", "azure"] + + def _extra_setup(self, **kwargs): + self._herbie_save_dir = tempfile.TemporaryDirectory() + + def __del__(self): + self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py new file mode 100644 index 00000000..780bcd1e --- /dev/null +++ b/src/geodata/datasets/hrrr/wind.py @@ -0,0 +1,35 @@ +# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from herbie import Herbie + +from ._base import HRRRBaseDataset + + +class HRRRWindDataset(HRRRBaseDataset): + """ + HRRRWindDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + + The HRRR dataset is a high-resolution weather forecast model that provides + hourly data for the United States. This dataset is useful for a variety of + applications, including renewable energy forecasting, weather prediction, + and climate research. + + This class provides a simple interface for downloading and processing the + HRRR dataset. It allows users to specify the years and months of interest, + as well as the variables they wish to download. + """ diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py new file mode 100644 index 00000000..563e8cb7 --- /dev/null +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -0,0 +1,82 @@ +# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import logging + +from multiprocessing import cpu_count +from unittest.mock import patch + + +import pandas as pd +import xarray as xr +from herbie import Herbie +from tqdm.auto import tqdm + +from ._base import HRRRBaseDataset + +logger = logging.getLogger(__name__) + + +def fake_print(*args, **kwargs): + pass + + +class HRRRWindSolarDataset(HRRRBaseDataset): + """HRRRWindSolarDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "wind_solar" + variables = ":[UV]GRD:[1,8]0 m" + + def download(self): + """Download the dataset from the HRRR dataset.""" + logger.info(f"Downloading {self.weather_config} dataset") + logger.info(self._herbie_save_dir.name) + + with patch("builtins.print", fake_print): + for file in tqdm(self.catalog, desc="Downloading Data", dynamic_ncols=True): + hours = pd.date_range( + f"{file['year']}-{file['month']}-01", + f"{file['year']}-{file['month']}-31", + freq="1h", + ) + + dss = [] + for hour in hours: + h = Herbie( + hour, + fxx=0, + product="sfc", + model="hrrr", + priority=self._priority, + save_dir=self._herbie_save_dir.name, + max_threads=cpu_count() * 2, + ) + logger.info(f"Downloading {hour}") + dss.append( + xr.concat(h.xarray(self.variables), dim="heightAboveGround") + ) + + ds = xr.concat(dss, dim="time").to_netcdf(file["save_path"]) + ds.close() + + logger.info(f"Downloaded {self.weather_config} dataset") + + @property + def downloaded(self): + pass diff --git a/uv.lock b/uv.lock index 7e5888c6..df3da842 100644 --- a/uv.lock +++ b/uv.lock @@ -370,6 +370,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, ] +[[package]] +name = "cfgrib" +version = "0.9.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "eccodes" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/d9/274599a790dfc384d0a06a849adfbed0c924ec5376eda189e503325e7e3f/cfgrib-0.9.14.1.tar.gz", hash = "sha256:a6e66e8a3d8f9823d3eef0c2c6ebca602d5bcc324f0baf4f3d13f68b0b40501e", size = 6510867 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/c6/7e8a2b07d2404a79b2cea577962f88e02662ba2b85e073b6e5eed8081878/cfgrib-0.9.14.1-py3-none-any.whl", hash = "sha256:0714ece262231b0d4006fc7ba5a04f287a9fd42473ac3f6ed4703eb2e7e92161", size = 48681 }, +] + [[package]] name = "cftime" version = "1.6.4" @@ -657,6 +672,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] +[[package]] +name = "eccodes" +version = "2.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi" }, + { name = "findlibs" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/76/bb33d761b1fa7a642e29c5e279feabf6f124b15c6eed64d012a30bf5c1ec/eccodes-2.39.0.tar.gz", hash = "sha256:0bf32c1f32c00d6c12091344bb5917d68f94cbc76625e66b16055784dbbc5c03", size = 2266990 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/63/33acf0c01567fdde5ea5baf92f085bdfd4961af3496476a03fdc4549a610/eccodes-2.39.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:6f1d637fe07fa2ff80954c442b9f63c3441cdadc2a561772bd3c988264ead27c", size = 6486940 }, + { url = "https://files.pythonhosted.org/packages/47/e3/b1298b31926a35f06af6ea0c4ae958b7ef5d3511c242b7fa34dd05df9dda/eccodes-2.39.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:d436eb91c21bb5697956083ca2b8b25fb5847a144ccd84391c5ffea1650f8f84", size = 6580698 }, + { url = "https://files.pythonhosted.org/packages/84/c8/d6c5eaf6a4c43b08265c6a7a67d1f4bcdae6c4bffc1102f51a902e0b6f51/eccodes-2.39.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1cd13926e94226de2517002cab401823cd469aa05baf0cae6b8ef086a44ca58a", size = 7329463 }, + { url = "https://files.pythonhosted.org/packages/b0/aa/1a5c386f13ab8df9ee787a08b0e35f619ad8b4e65b1b7e9ec252ea0f7b40/eccodes-2.39.0-cp310-cp310-win_amd64.whl", hash = "sha256:3e59e1f33e6572eae6348e5b49b6de081a7a7cd1601fa0c31273f84f29d1ae9a", size = 6157549 }, + { url = "https://files.pythonhosted.org/packages/3f/f2/7355a83be71880cc8c41d6093bf7e7fe56e75255080a692a7751e78b6f40/eccodes-2.39.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:e12720dff21c952fc5706c58639e6a4f9ab04afe549d7d9ae34a5a6975c80c4a", size = 6486941 }, + { url = "https://files.pythonhosted.org/packages/0c/51/34070e1d4ecdd2a162a351f48e50750f8677a73aabf80669fcf61a2189a0/eccodes-2.39.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:9d55059ba468cb014ecbc8df4186a34ad80ad76e840859482f01e798dcb1a6a4", size = 6580697 }, + { url = "https://files.pythonhosted.org/packages/8b/cc/64bf04c8873278bd7561ac5492651a293dbda9993c9119586edbc543c7de/eccodes-2.39.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ad098a71880ee1496db95255c5f8aac246035eed508719251e460a375d759231", size = 7329478 }, + { url = "https://files.pythonhosted.org/packages/4e/4a/e7a6b62b33c7d541906ecd54252586befe4123ad3ab126c7eda95af08476/eccodes-2.39.0-cp311-cp311-win_amd64.whl", hash = "sha256:a23091b75610981ce1c677d36d3d07ab9f646f3ede25ef12030e2ba93b57211b", size = 6157548 }, + { url = "https://files.pythonhosted.org/packages/2d/42/d8e0f51dee957c66acbf32803f086cdc1d5485090e6eb52f88450ec304c6/eccodes-2.39.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:9d49b6bda883051c0dd9b23bd8fd79849b50801139ce5841ff9b8acbc11cccef", size = 6486942 }, + { url = "https://files.pythonhosted.org/packages/1b/6f/fde312acec0fcebc6253c2e1458d39f3782bd8c92a5513287ebe7013e1ec/eccodes-2.39.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:6be54b2362f103e0a4bb2c8f7a487de52d5813e9eb0de153a1d6b88e3d306329", size = 6580698 }, + { url = "https://files.pythonhosted.org/packages/71/46/1f176bbb20c46f127dc34f260b5d633c8ca4aef9835f71407f33410430cc/eccodes-2.39.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:772e0836e4a99777c71a122abd047c11c7b0598b7f574591ebe277310952623d", size = 7329664 }, + { url = "https://files.pythonhosted.org/packages/a1/59/4c2c19b9604971309b050e19836193a79405987917430409698e7a8ff55a/eccodes-2.39.0-cp312-cp312-win_amd64.whl", hash = "sha256:976a5fbfb74612cc79742aa8d54dbfa384c090d7c6574b6360f09665938141b7", size = 6157548 }, + { url = "https://files.pythonhosted.org/packages/c2/a4/fb4c7e2f7df631ff2cfdf345f6978452c00bdc24b0c9eff33081d6d533f4/eccodes-2.39.0-py3-none-any.whl", hash = "sha256:39f63f09a93b33dfdd805a41b88486636259c113955e5176327d3a97ce2ef0a7", size = 43196 }, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -684,6 +726,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/ca/086311cdfc017ec964b2436fe0c98c1f4efcb7e4c328956a22456e497655/fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a", size = 23543 }, ] +[[package]] +name = "findlibs" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/be/6c72ef9d990cd25fe3dd97ebe9d77a859f7d27b7273e62ad750846d207ee/findlibs-0.0.5.tar.gz", hash = "sha256:7a801571e999d0ee83f9b92cbb598c21f861ee26ca9dba74cea8958ba4335e7e", size = 6581 } + [[package]] name = "fonttools" version = "4.54.1" @@ -777,6 +825,7 @@ docs = [ ] download = [ { name = "cdsapi" }, + { name = "herbie-data" }, ] notebook = [ { name = "notebook" }, @@ -797,6 +846,7 @@ requires-dist = [ { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.3" }, { name = "dask", specifier = ">=2024.9.0" }, { name = "geopandas", specifier = ">=1.0.1" }, + { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2024.8.0" }, { name = "matplotlib", specifier = "==3.9.2" }, { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "netcdf4", specifier = ">=1.7.1.post2" }, @@ -905,6 +955,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, ] +[[package]] +name = "herbie-data" +version = "2024.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgrib" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pygrib" }, + { name = "requests" }, + { name = "toml" }, + { name = "xarray" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/59/136b443a12073546ad7987db6d78dd43d8550689134145be4bbfac0d0a57/herbie_data-2024.8.0.tar.gz", hash = "sha256:83831205ea415b6f245d829cc13a162263074035e5e365c38f239ee04e3d6c14", size = 102912 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0a/9bd4412d9fe1c30e26b58f4dbec64d39276110f84bf6316d12242b2f1296/herbie_data-2024.8.0-py3-none-any.whl", hash = "sha256:196ecc028dca71c99ffb7452d8a443a64b57c27605f70ac08f5810f3d606088b", size = 100959 }, +] + [[package]] name = "httpcore" version = "1.0.6" @@ -2073,6 +2141,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513 }, ] +[[package]] +name = "pygrib" +version = "2.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "pyproj" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/60/bac29fc06197f85efccb346879da3ae9ee125525d43e28b0a1b768831a74/pygrib-2.1.6.tar.gz", hash = "sha256:047980aeb010ef457999950bcc8e46556910316cb77fe78c0bd1b3520aa920f0", size = 21808824 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/11/55388ae3f0abce352942409ee687d74348fd86f54b60a9517f2311194fc3/pygrib-2.1.6-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:354028fcfc5f29dbebe16da2df3c8cf41383818cf27a43adfa6cc1a4e43c249b", size = 18472537 }, + { url = "https://files.pythonhosted.org/packages/ec/33/9395540f48099d6b04e9f583cc5428ebcb56d54beffbed9207d671d14826/pygrib-2.1.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:06b3fb25889225a599a908fdb8d4590ab30a2738048d877725f90b6a2a777983", size = 18358507 }, + { url = "https://files.pythonhosted.org/packages/70/4e/741e1f5fa63a08a98de12bf00f274249cd5c3f9296af11f11c16e4375d06/pygrib-2.1.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37ce2c761489b68f2642c739a488470b58aacf13c96bdae671dd35730960e94e", size = 18638885 }, + { url = "https://files.pythonhosted.org/packages/50/5d/12ae450349ce3d51844e775dff0ee1b04a9aee963d6ed6102ced290805b4/pygrib-2.1.6-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:d6b912cc528c3a87e4f9f990d5c1e57a54f61835dad5b8daac6adb048faf5c5a", size = 18473599 }, + { url = "https://files.pythonhosted.org/packages/53/9a/a1742fa64b2702d7723c2595bf1b36b1c56fd1915f6e3f185f057a79fbc3/pygrib-2.1.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:dc7fcb20bd5e2a94e1b7501bed45d418eadc6e325bed5f4de37a254409b5a5fe", size = 18358366 }, + { url = "https://files.pythonhosted.org/packages/c8/f0/9490a3bf86feef1be2678a6ce15131d44f6f06c863c2462f7f4a52c35c0a/pygrib-2.1.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69a42b44ae395280b08f62e598b696444446759ae4954db42ad2d01ee2c94d8b", size = 18633622 }, + { url = "https://files.pythonhosted.org/packages/71/99/436f5af4e9093277b13153c6ce51336df886e7493076c3d756c44c988d47/pygrib-2.1.6-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:fa4613eef98b26bf9a8719869f656aa15362f700311aec777b2190deb1060a1d", size = 18467761 }, + { url = "https://files.pythonhosted.org/packages/b0/4d/e2d3150961801d46af88c4cf113c00ff0825206ac6d726cec11f667c9f8e/pygrib-2.1.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:332d29bbc16749375b24c8d9bd01180c0a9eda471df86b2073432fba3339d658", size = 18356620 }, + { url = "https://files.pythonhosted.org/packages/ab/cc/4b2b85241086a3c42fcf4e9e225a1b2f5bfcb299a4c9c8e46f56c6a2d7c3/pygrib-2.1.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d368c359acacae4061ebcdfe635fb826f21fbc1f6c2227a269c870d9e8b49e70", size = 18617948 }, + { url = "https://files.pythonhosted.org/packages/ac/bb/a35a9b012c234416100abe53661cb7002e8fd6f3120f4f85475604cf9d00/pygrib-2.1.6-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:3cf2d87ad30449726d9279ca8cfa4afbdc83670750a4dfb5d965e3fc55d3aa8d", size = 18466301 }, + { url = "https://files.pythonhosted.org/packages/fd/15/bb162b016378993b5a779d83ec499dfd82d8df7776885fb826b69d2b7b98/pygrib-2.1.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:00f84e99372c6cf51f6a67e26c394dc3371ec356906e5aa9177f12372d92b228", size = 18356130 }, + { url = "https://files.pythonhosted.org/packages/ea/42/112c2f6836e730343fe21ad85e916c1b742d1f951738a3e9b3f6fab65128/pygrib-2.1.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56d70f492e1a298429a94662c5003c6959d8cc1b078173312fa9cf3bf36a14b8", size = 18613954 }, +] + [[package]] name = "pyogrio" version = "0.10.0" @@ -2894,6 +2987,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/4d/0db5b8a613d2a59bbc29bc5bb44a2f8070eb9ceab11c50d477502a8a0092/tinycss2-1.3.0-py3-none-any.whl", hash = "sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7", size = 22532 }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] + [[package]] name = "tomli" version = "2.0.2" From 7aef6fa061eb420356d94193ad6eee0d600d22c3 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sat, 4 Jan 2025 11:57:18 -0800 Subject: [PATCH 02/54] fix: update cdsapi dependency version --- pyproject.toml | 2 +- src/geodata/datasets/_base.py | 60 ++++++++++++++----- .../datasets/era5/hourly/wind_solar.py | 6 +- src/geodata/logging.py | 6 +- uv.lock | 44 +++++++------- 5 files changed, 75 insertions(+), 43 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 195652c0..1bc5f6f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ license = {text = "GPLv3"} [project.optional-dependencies] download = [ - "cdsapi>=0.7.3", + "cdsapi>=0.7.5", "herbie-data>=2024.8.0", ] notebook = [ diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 7cb02cf9..68521609 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -137,13 +137,18 @@ def _extra_setup(self, **kwargs): """ @property - @abc.abstractmethod def downloaded(self): """A boolean flag indicating whether the dataset has been prepared for use. This typically means that the dataset has been downloaded, preprocessed, and stored in a format that is ready for use. + + The basic implementation of this method checks the presence of each file in + the catalog in the storage root. Subclasses can override this behavior if + a more comprehensive check is required. """ + return all((file["save_path"].exists() for file in self.catalog)) + @abc.abstractmethod def _download_file(self, file: dict): """Method to download a single file from the dataset. This method @@ -159,25 +164,33 @@ def _download_file(self, file: dict): - save_path: the path where the file should be saved """ - def download(self): + def download(self, force: bool = False): """Method to download the dataset. This method should download the dataset files and store them in the appropriate location. + + Args: + force: A boolean flag indicating whether to force the download of + the dataset, even if it has already been downloaded. """ + if self.downloaded and not force: + logger.info(f"{self} has already been downloaded.") + return + for file in tqdm( self.catalog, desc="Downloading", unit="file", dynamic_ncols=True ): self._download_file(file) def __repr__(self): - return "".format( + return "".format( self.module, self.weather_config, self.years.start, self.years.stop, self.months.start, self.months.stop, - "Prepared" if self.prepared else "Unprepared", + "Downloaded" if self.downloaded else "Not Downloaded", " " + self.extra_repr if self.extra_repr else "", ) @@ -206,34 +219,46 @@ def catalog(self): match self.frequency: case "monthly": - yield from self._monthly_catalog() + return self._monthly_catalog() case "daily": - yield from self._daily_catalog() + return self._daily_catalog() case "hourly": - yield from self._hourly_catalog() + return self._hourly_catalog() case _: raise ValueError( f"Invalid frequency {self.frequency} defined for this dataset." ) def _monthly_catalog(self): + catalog = [] + for year, month in itertools.product( range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), ): save_path = self.storage_root / f"{year}_{month:02d}.nc" - yield {"year": year, "month": month, "save_path": save_path} + catalog.append({"year": year, "month": month, "save_path": save_path}) + + return catalog def _daily_catalog(self): + catalog = [] + for year, month in itertools.product( range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), ): for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): save_path = self.storage_root / f"{year}_{month:02d}_{day:02d}.nc" - yield {"year": year, "month": month, "day": day, "save_path": save_path} + catalog.append( + {"year": year, "month": month, "day": day, "save_path": save_path} + ) + + return catalog def _hourly_catalog(self): + catalog = [] + for year, month in itertools.product( range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), @@ -244,10 +269,13 @@ def _hourly_catalog(self): self.storage_root / f"{year}_{month:02d}_{day:02d}_{hour:02d}.nc" ) - yield { - "year": year, - "month": month, - "day": day, - "hour": hour, - "save_path": save_path, - } + catalog.append( + { + "year": year, + "month": month, + "day": day, + "hour": hour, + "save_path": save_path, + } + ) + return catalog diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py index be14caee..11da5f94 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -15,6 +15,7 @@ import logging import os +import pprint import tempfile import zipfile from pathlib import Path @@ -90,6 +91,8 @@ def _download_file(self, file: dict): if self.bounds is not None: full_request["area"] = self.bounds[::-1] + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + full_result = self.client.retrieve(self.product, full_request) if full_result.content_type == "application/zip": logger.info( @@ -114,6 +117,3 @@ def _download_file(self, file: dict): logger.info("Preprocessing complete with zipfile") logger.info("Successfully downloaded to %s", save_path) - - def downloaded(self): - pass diff --git a/src/geodata/logging.py b/src/geodata/logging.py index a3bab789..f1682fbb 100644 --- a/src/geodata/logging.py +++ b/src/geodata/logging.py @@ -71,7 +71,11 @@ def format(self, record): logger = _logging.getLogger("geodata") -logger.setLevel(_logging.INFO) + +# Only set the level if it hasn't been set yet +if logger.level == _logging.NOTSET: + logger.setLevel(_logging.INFO) + logger.propagate = False if not logger.hasHandlers(): ch = _logging.StreamHandler() diff --git a/uv.lock b/uv.lock index df3da842..9e0125dc 100644 --- a/uv.lock +++ b/uv.lock @@ -275,33 +275,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/96/b100d19b55acc2b3cf41c52f31804533534de6e9da445ba11d83fa262d5d/Bottleneck-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1339b9ad3ee217253f246cde5c3789eb527cf9dd31ff0a1f5a8bf7fc89eadad", size = 111760 }, ] -[[package]] -name = "cads-api-client" -version = "1.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "multiurl" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/d5/54e3f4153e07b053b4e3b1576fc385c33263f40ca5030d9b3ef07013d4d0/cads_api_client-1.4.3.tar.gz", hash = "sha256:52359ea743a84b597cc589730ff4b414707bce4e5e2279835528b2d1273c19b8", size = 40795 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/87/bece1e563ce5602a2cd6b92e6b870fa8ad45a9c21866e31326385c766efd/cads_api_client-1.4.3-py3-none-any.whl", hash = "sha256:8f2db934da8f2c63902bf808f33eb2e76734feed284783acd2086a72dd4d6833", size = 24332 }, -] - [[package]] name = "cdsapi" -version = "0.7.3" +version = "0.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cads-api-client" }, + { name = "datapi" }, { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/50/b8/aaade2b628ea5784f637a0c2762565a0fb9f0b121a890aa6461e06b70659/cdsapi-0.7.3.tar.gz", hash = "sha256:883a1376ca495457eb55fd548dbbb6f5b64f2e4c880b3586dd37ba9041e51c82", size = 13121 } +sdist = { url = "https://files.pythonhosted.org/packages/3b/62/81b38105ef75486308179d510360c1aa251ce0b7d17fe88c0e1de05c6c94/cdsapi-0.7.5.tar.gz", hash = "sha256:55221c573b8cefe83cc0bfe01a3d31213c82bf9acce70455350dd24b8095c23a", size = 13188 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/2d/bfeabe547fc186471b49b9df384f44c181fe9f41899d2cf13f5bda195af9/cdsapi-0.7.3-py2.py3-none-any.whl", hash = "sha256:3bf432783e6ff0b47b0b33466c6e05e7ddad52fda5f05bf269596f5be30d623b", size = 12140 }, + { url = "https://files.pythonhosted.org/packages/4e/c4/49f01f1382d449581d5d3db0fa49ccc23a1b4f91d615108b631c7cff40cd/cdsapi-0.7.5-py2.py3-none-any.whl", hash = "sha256:8586b837aea89ceeae379b388fbb0ace0a19b94b221f731c65632417007f69fb", size = 12201 }, ] [[package]] @@ -624,6 +609,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ea/3df533d551bc673f8a295450a8e28707e980fd3b55117edb1d1aa4cc374d/dask-2024.9.1-py3-none-any.whl", hash = "sha256:3757bb6c976f0436fef6bd6ad32f8983ee5ce7d8a738a1f643e208cd390ec794", size = 1257378 }, ] +[[package]] +name = "datapi" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "multiurl" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/95/e4abd8958a15793a9d6df1d6f0bcff2ac245901d9cc44dfb14be3bbb1bf6/datapi-0.1.1.tar.gz", hash = "sha256:7526169c61d103bd8585a54778aa234ba89ef6f6093d2079db9d089ad5e448e0", size = 43438 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/a2/8f4f500bfa3b407afdd3bd5987dd0d5f0b1dcfb9cfd88a95a753bd76c71a/datapi-0.1.1-py3-none-any.whl", hash = "sha256:3c8e9c7c5a29b9b392633d9527266080868d3f411b78e05acac4cd9304599bd9", size = 26899 }, +] + [[package]] name = "debugpy" version = "1.8.6" @@ -843,7 +843,7 @@ dev = [ requires-dist = [ { name = "boto3", specifier = "==1.26.46" }, { name = "bottleneck", specifier = ">=1.3.6" }, - { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.3" }, + { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.5" }, { name = "dask", specifier = ">=2024.9.0" }, { name = "geopandas", specifier = ">=1.0.1" }, { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2024.8.0" }, @@ -1601,7 +1601,7 @@ wheels = [ [[package]] name = "multiurl" -version = "0.3.1" +version = "0.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, @@ -1609,7 +1609,7 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cc/12/4e979f71d90ca5625647f93cd484c733a7e8ae4fd9f6d15369613d727301/multiurl-0.3.1.tar.gz", hash = "sha256:c7001437b59d56d4c310d725c3dcfff98c97c4b652893d88989853827465d442", size = 18161 } +sdist = { url = "https://files.pythonhosted.org/packages/33/db/aad981174d3bdaecc1d7e1f2d176641f022300ddf3a17b13c2775d041b7a/multiurl-0.3.3.tar.gz", hash = "sha256:f4d0b69dcf4a0ed740daa313dbcd4d5665420d305c50ca879285e96dc828093f", size = 18382 } [[package]] name = "mypy-extensions" From e13ca1f7fbfb6a8d92db3c63611d2201df7573e1 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:41:37 -0800 Subject: [PATCH 03/54] wip: MERRA2 dataset --- .../datasets/{merra2.py => _merra2.py} | 0 src/geodata/datasets/era5/__init__.py | 19 +++ .../datasets/era5/monthly/wind_solar.py | 101 ++++++++++++ src/geodata/datasets/hrrr/_base.py | 2 + src/geodata/datasets/merra2/_base.py | 147 ++++++++++++++++++ .../datasets/merra2/hourly/surface_flux.py | 42 +++++ 6 files changed, 311 insertions(+) rename src/geodata/datasets/{merra2.py => _merra2.py} (100%) create mode 100644 src/geodata/datasets/era5/__init__.py create mode 100644 src/geodata/datasets/era5/monthly/wind_solar.py create mode 100644 src/geodata/datasets/merra2/_base.py create mode 100644 src/geodata/datasets/merra2/hourly/surface_flux.py diff --git a/src/geodata/datasets/merra2.py b/src/geodata/datasets/_merra2.py similarity index 100% rename from src/geodata/datasets/merra2.py rename to src/geodata/datasets/_merra2.py diff --git a/src/geodata/datasets/era5/__init__.py b/src/geodata/datasets/era5/__init__.py new file mode 100644 index 00000000..76d5add4 --- /dev/null +++ b/src/geodata/datasets/era5/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .hourly.wind_solar import ERA5WindSolarHourlyDataset +from .monthly.wind_solar import ERA5WindSolarMonthlyDataset + +__all__ = ["ERA5WindSolarHourlyDataset", "ERA5WindSolarMonthlyDataset"] diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py new file mode 100644 index 00000000..fbff555a --- /dev/null +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -0,0 +1,101 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import pprint +import tempfile +import zipfile +from pathlib import Path + +import xarray as xr + +from ..hourly.wind_solar import ERA5WindSolarHourlyDataset + +logger = logging.getLogger(__name__) + + +class ERA5WindSolarMonthlyDataset(ERA5WindSolarHourlyDataset): + """ERA5WindSolarMonthlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind and solar + information. This dataset is stored in monthly intervals. + + The ERA5 dataset is a reanalysis dataset that provides a comprehensive + record of the Earth's climate. It is produced by the European Centre for + Medium-Range Weather Forecasts (ECMWF) and is available from 1980 to + present. + + Note: + - The specific variables that are downloaded are: + - 100m_u_component_of_wind + - 100m_v_component_of_wind + - 2m_temperature + - runoff + - soil_temperature_level_4 + - surface_net_solar_radiation + - surface_pressure + - surface_solar_radiation_downwards + - toa_incident_solar_radiation + - total_sky_direct_solar_radiation_at_surface + - forecast_surface_roughness + - geopotential + """ + + weather_config = "wind_solar_monthly" + + def _download_file(self, file: dict): + year: int = file["year"] + month: int = file["month"] + save_path: Path = file["save_path"] + + full_request = { + "product_type": self.product_type, + "format": "netcdf", + "variable": list(self.variables.keys()), + "year": year, + "month": month, + "day": [f"{d:02d}" for d in range(1, 32)], + "time": "00:00", + } + + if self.bounds is not None: + full_request["area"] = self.bounds[::-1] + + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + + full_result = self.client.retrieve(self.product, full_request) + if full_result.content_type == "application/zip": + logger.info( + "Multiple files found with request. Additional unzipping/preprocessing needed." + ) + + with tempfile.TemporaryDirectory() as tempdir: + full_result.download(os.path.join(tempdir, "download.zip")) + with zipfile.ZipFile( + os.path.join(tempdir, "download.zip"), "r" + ) as zip_ref: + zip_ref.extractall(tempdir) + + with xr.open_mfdataset( + [ + os.path.join(tempdir, f) + for f in os.listdir(tempdir) + if f.endswith(".nc") + ] + ) as ds: + ds.to_netcdf(save_path) + + logger.info("Preprocessing complete with zipfile") + logger.info("Successfully downloaded to %s", save_path) diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index cd5f8cd2..afa5d672 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -27,6 +27,8 @@ class HRRRBaseDataset(BaseDataset): """HRRRBaseDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. + + TODO: Support multi-file downloads """ module = "hrrr" diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py new file mode 100644 index 00000000..e475ec67 --- /dev/null +++ b/src/geodata/datasets/merra2/_base.py @@ -0,0 +1,147 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from pathlib import Path + +import numpy as np +import xarray as xr +import requests + +from .._base import BaseDataset + + +class MERRA2BaseDataset(BaseDataset): + """MERRA2BaseDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + module = "merra2" + url_template = "" + + def _download_file(self, file: dict): + assert "url" in file, "URL is required to download the file" + + url: str = file["url"] + path: Path = file["path"] + + # Download the file + with requests.get(url, stream=True) as r: + r.raise_for_status() + + with open(path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + + def spinup_year(year: int, month: int): + """Returns the spinup period for the given year and month. + See https://gmao.gsfc.nasa.gov/pubs/docs/Bosilovich785.pdf for more + information. + + Args: + year (int): The year of the dataset + month (int): The month of the dataset + + Returns: + str: The spinup period + """ + if year >= 1980 and year < 1992: + spinup = "100" + elif year >= 1992 and year < 2001: + spinup = "200" + elif year >= 2001 and year < 2011: + spinup = "300" + elif year >= 2011 and year < 2020: + spinup = "400" + elif year == 2020 and month == 9: + spinup = "401" + else: + spinup = "400" + + return spinup + + def convert_and_subset_lons_lats_merra2( + ds: xr.Dataset | xr.DataArray, xs: slice, ys: slice + ): + """Rename geographic dimensions to x,y. Subset x,y according to xs, ys. + + Args: + ds (xr.Dataset | xr.DataArray): The dataset to subset + xs (slice): The slice of longitudes to subset + ys (slice): The slice of latitudes to subset + + Returns: + xr.Dataset | xr.DataArray: The subsetted dataset + """ + + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(lat=ys) + + # Longitudes should go from -180. to +180. + if len(ds.coords["lon"].sel(lon=slice(xs.start + 360.0, xs.stop + 360.0))): + ds = xr.concat( + [ds.sel(lon=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(lon=xs)], + dim="lon", + ) + ds = ds.assign_coords( + lon=np.where( + ds.coords["lon"].values <= 180, + ds.coords["lon"].values, + ds.coords["lon"].values - 360.0, + ) + ) + else: + ds = ds.sel(lon=xs) + + ds = ds.rename({"lon": "x", "lat": "y"}) + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + + return ds + + def _rename_and_clean_coords( + ds: xr.Dataset | xr.DataArray, add_lon_lat: bool = True + ): + """Rename 'longitude' and 'latitude' columns to 'x' and 'y' + + Optionally `add_lon_lat` preserves latitude and longitude + columns as 'lat' and 'lon'. + + Args: + ds (xr.Dataset): The dataset to rename + add_lon_lat (bool, optional): Whether to preserve latitude and longitude columns. Defaults to True. + """ + + ds = ds.rename({"lon": "x", "lat": "y"}) + if add_lon_lat: + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + return ds + + def _hourly_catalog(self): + if not self.url_template: + raise NotImplementedError("url_template is not defined for this dataset") + + catalog = super()._hourly_catalog() + + for file in catalog: + file["spinup"] = self.spinup_year(file["year"], file["month"]) + file["url"] = self.url_template.format(**file) + + return catalog diff --git a/src/geodata/datasets/merra2/hourly/surface_flux.py b/src/geodata/datasets/merra2/hourly/surface_flux.py new file mode 100644 index 00000000..5f8a4af4 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/surface_flux.py @@ -0,0 +1,42 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxHourlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_hourly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + ] + url_template = "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4" From 78e875e88b0aa96f44efde5e4951dd8cb9525368 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:13:06 -0800 Subject: [PATCH 04/54] feat: add some functionalities to base datasets class --- src/geodata/datasets/_base.py | 92 +++++++++++++++++++++++++--- src/geodata/datasets/hrrr/wind.py | 1 - src/geodata/datasets/merra2/_base.py | 30 ++------- src/geodata/types.py | 20 ++++++ 4 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 src/geodata/types.py diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 68521609..702b7e96 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -1,4 +1,4 @@ -# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -20,9 +20,11 @@ from pathlib import Path import pandas as pd +import xarray as xr from tqdm.auto import tqdm from ..config import DATASET_ROOT_PATH +from ..types import BoundRange, DateRange logger = logging.getLogger(__name__) @@ -60,9 +62,9 @@ class BaseDataset(abc.ABC): def __init__( self, - years: Sequence[int] | slice, - months: Sequence[int] | slice, - bounds: Sequence[int] | None = None, + years: DateRange, + months: DateRange, + bounds: BoundRange | None = None, **kwargs, ): if not hasattr(self, "module"): @@ -136,6 +138,20 @@ def _extra_setup(self, **kwargs): that is required for the dataset. """ + def apply(self, func: callable, *args, **kwargs): + """Method to apply a function to each file in the dataset. + + Args: + func: A function that takes an xarray.Dataset as its first argument. + *args: Additional arguments to pass to the function. + **kwargs: Additional keyword arguments to pass to the function. + """ + + for file in self.catalog: + with xr.open_dataset(file["save_path"], chunks="auto") as ds: + ds = func(ds, *args, **kwargs) + ds.to_netcdf(file["save_path"]) + @property def downloaded(self): """A boolean flag indicating whether the dataset has been prepared @@ -177,11 +193,36 @@ def download(self, force: bool = False): logger.info(f"{self} has already been downloaded.") return - for file in tqdm( - self.catalog, desc="Downloading", unit="file", dynamic_ncols=True - ): + for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): self._download_file(file) + logger.info(f"Downloaded {self}") + logger.info("Cleaning and renaming coordinates") + + self.apply(self._rename_and_clean_coords) + + @apply + def trim_variables( + self, variables: Sequence[str] | None = None, **kwargs + ) -> xr.Dataset | xr.DataArray: + """Method to trim the dataset to only include the specified variables. + + Args: + variables: A sequence of strings representing the variables to keep. + If None, we will keep the variables specified in the `variables` + attribute of the dataset. + """ + + if variables is None: + if not hasattr(self, "variables"): + raise ValueError( + "The dataset does not have a `variables` attribute defined." + "Please specify the variables to keep." + ) + variables: Sequence[str] = getattr(self, "variables") + + return kwargs["ds"][variables] + def __repr__(self): return "".format( self.module, @@ -194,6 +235,13 @@ def __repr__(self): " " + self.extra_repr if self.extra_repr else "", ) + @property + @abc.abstractmethod + def projection(self): + """The projection of the dataset. This should be a string that + represents the projection of the dataset. + """ + @property def extra_repr(self): return "" @@ -279,3 +327,33 @@ def _hourly_catalog(self): } ) return catalog + + def _rename_and_clean_coords( + ds: xr.Dataset | xr.DataArray, add_lon_lat: bool = True + ): + """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' + + Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. + + Args: + ds (xarray.Dataset): Dataset to rename + add_lon_lat (bool, optional): Add lon/lat columns. Defaults to True. + + Returns: + xarray.Dataset: Dataset with renamed coordinates + """ + + # Rename latitude / lat -> y, longitude / lon -> x + if "latitude" in list(ds.coords): + ds = ds.rename({"latitude": "y"}) + if "longitude" in list(ds.coords): + ds = ds.rename({"longitude": "x"}) + if "lat" in list(ds.coords): + ds = ds.rename({"lat": "y"}) + if "lon" in list(ds.coords): + ds = ds.rename({"lon": "x"}) + + if add_lon_lat: + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + + return ds diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py index 780bcd1e..1b43c5fd 100644 --- a/src/geodata/datasets/hrrr/wind.py +++ b/src/geodata/datasets/hrrr/wind.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from herbie import Herbie from ._base import HRRRBaseDataset diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index e475ec67..1e59b8e7 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -16,9 +16,10 @@ from pathlib import Path import numpy as np -import xarray as xr import requests +import xarray as xr +from ...types import CoordRange from .._base import BaseDataset @@ -73,8 +74,8 @@ def spinup_year(year: int, month: int): return spinup def convert_and_subset_lons_lats_merra2( - ds: xr.Dataset | xr.DataArray, xs: slice, ys: slice - ): + ds: xr.Dataset | xr.DataArray, xs: CoordRange, ys: CoordRange + ) -> xr.Dataset | xr.DataArray: """Rename geographic dimensions to x,y. Subset x,y according to xs, ys. Args: @@ -111,28 +112,7 @@ def convert_and_subset_lons_lats_merra2( else: ds = ds.sel(lon=xs) - ds = ds.rename({"lon": "x", "lat": "y"}) - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - - return ds - - def _rename_and_clean_coords( - ds: xr.Dataset | xr.DataArray, add_lon_lat: bool = True - ): - """Rename 'longitude' and 'latitude' columns to 'x' and 'y' - - Optionally `add_lon_lat` preserves latitude and longitude - columns as 'lat' and 'lon'. - - Args: - ds (xr.Dataset): The dataset to rename - add_lon_lat (bool, optional): Whether to preserve latitude and longitude columns. Defaults to True. - """ - - ds = ds.rename({"lon": "x", "lat": "y"}) - if add_lon_lat: - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - return ds + return super()._rename_and_clean_coords(ds) def _hourly_catalog(self): if not self.url_template: diff --git a/src/geodata/types.py b/src/geodata/types.py new file mode 100644 index 00000000..bc7a9583 --- /dev/null +++ b/src/geodata/types.py @@ -0,0 +1,20 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +DateRange = slice[int, int, None] | tuple[int, int] | list[int, int] +CoordRange = slice[float, float, None] | tuple[float, float] | list[float, float] +BoundRange = tuple[float, float, float, float] | list[float, float, float, float] + +__all__ = ["DateRange", "CoordRange"] From 2cf4580c3e9c7aa6cf72c79f6d921f502a1c587f Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:30:49 -0800 Subject: [PATCH 05/54] wip: MERRA2 dataset --- src/geodata/datasets/_base.py | 28 ++++++++++++++++++---------- src/geodata/datasets/era5/_base.py | 1 + src/geodata/datasets/merra2/_base.py | 10 ++++++---- src/geodata/types.py | 6 +++--- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 702b7e96..7c8ae7c6 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -132,13 +132,13 @@ def __init__( self._extra_setup(**kwargs) - @abc.abstractmethod def _extra_setup(self, **kwargs): """Method to be implemented by subclasses to handle any extra setup that is required for the dataset. """ - def apply(self, func: callable, *args, **kwargs): + @staticmethod + def apply(func: callable, *args, **kwargs): """Method to apply a function to each file in the dataset. Args: @@ -147,10 +147,13 @@ def apply(self, func: callable, *args, **kwargs): **kwargs: Additional keyword arguments to pass to the function. """ - for file in self.catalog: - with xr.open_dataset(file["save_path"], chunks="auto") as ds: - ds = func(ds, *args, **kwargs) - ds.to_netcdf(file["save_path"]) + def wrapper(self, *args, **kwargs): + for file in self.catalog: + with xr.open_dataset(file["save_path"], chunks="auto") as ds: + ds = func(ds, *args, **kwargs) + ds.to_netcdf(file["save_path"]) + + return wrapper @property def downloaded(self): @@ -163,7 +166,7 @@ def downloaded(self): a more comprehensive check is required. """ - return all((file["save_path"].exists() for file in self.catalog)) + return all((file["downloaded"] for file in self.catalog)) @abc.abstractmethod def _download_file(self, file: dict): @@ -267,16 +270,21 @@ def catalog(self): match self.frequency: case "monthly": - return self._monthly_catalog() + cat = self._monthly_catalog() case "daily": - return self._daily_catalog() + cat = self._daily_catalog() case "hourly": - return self._hourly_catalog() + cat = self._hourly_catalog() case _: raise ValueError( f"Invalid frequency {self.frequency} defined for this dataset." ) + for file in cat: + file["downloaded"] = file["save_path"].exists() + + return cat + def _monthly_catalog(self): catalog = [] diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index 41b20a4a..631fee09 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -27,6 +27,7 @@ class ERA5BaseDataset(BaseDataset): """ module = "era5" + projection = "latlong" def _extra_setup(self, **kwargs): self.logger = logging.getLogger(__name__.replace("._base", ".client")) diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 1e59b8e7..20a77923 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -30,13 +30,15 @@ class MERRA2BaseDataset(BaseDataset): """ module = "merra2" + projection = "latlong" + frequency = "daily" url_template = "" def _download_file(self, file: dict): assert "url" in file, "URL is required to download the file" url: str = file["url"] - path: Path = file["path"] + path: Path = file["save_path"] # Download the file with requests.get(url, stream=True) as r: @@ -46,7 +48,7 @@ def _download_file(self, file: dict): for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - def spinup_year(year: int, month: int): + def spinup_year(self, year: int, month: int): """Returns the spinup period for the given year and month. See https://gmao.gsfc.nasa.gov/pubs/docs/Bosilovich785.pdf for more information. @@ -114,11 +116,11 @@ def convert_and_subset_lons_lats_merra2( return super()._rename_and_clean_coords(ds) - def _hourly_catalog(self): + def _daily_catalog(self): if not self.url_template: raise NotImplementedError("url_template is not defined for this dataset") - catalog = super()._hourly_catalog() + catalog = super()._daily_catalog() for file in catalog: file["spinup"] = self.spinup_year(file["year"], file["month"]) diff --git a/src/geodata/types.py b/src/geodata/types.py index bc7a9583..35fa48ef 100644 --- a/src/geodata/types.py +++ b/src/geodata/types.py @@ -13,8 +13,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -DateRange = slice[int, int, None] | tuple[int, int] | list[int, int] -CoordRange = slice[float, float, None] | tuple[float, float] | list[float, float] -BoundRange = tuple[float, float, float, float] | list[float, float, float, float] +DateRange = slice | tuple[int, int] | list[int] +CoordRange = slice | tuple[float, float] | list[float] +BoundRange = tuple[float, float, float, float] | list[float] __all__ = ["DateRange", "CoordRange"] From f1ef222ebe594f030fc19c7037febc9f243858dd Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:57:34 -0800 Subject: [PATCH 06/54] feat: draft impl of HRRR hourly wind dataset for CONUS --- src/geodata/datasets/_base.py | 4 +- src/geodata/datasets/hrrr/_base.py | 3 +- src/geodata/datasets/hrrr/wind.py | 66 ++++++++++++++++++++++++++-- src/geodata/datasets/merra2/_base.py | 2 + src/geodata/logging.py | 53 +++++++++++++++++++++- 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 7c8ae7c6..3d3ecce9 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -125,8 +125,8 @@ def __init__( ) if not self.storage_root.exists(): logger.info( - f"""Storage directory for {self.__class__.__name__} - does not exist, creating now at {self.storage_root}""" + f"Storage directory for {self.__class__.__name__} does not exist, " + "creating now at {self.storage_root}" ) self.storage_root.mkdir(parents=True) diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index afa5d672..bba25071 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -27,11 +27,10 @@ class HRRRBaseDataset(BaseDataset): """HRRRBaseDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. - - TODO: Support multi-file downloads """ module = "hrrr" + projection = "latlong" _priority = ["google", "aws", "azure"] def _extra_setup(self, **kwargs): diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py index 1b43c5fd..d79ee1b4 100644 --- a/src/geodata/datasets/hrrr/wind.py +++ b/src/geodata/datasets/hrrr/wind.py @@ -1,4 +1,4 @@ -# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,13 +13,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging +import multiprocessing as mp +import xarray as xr +import pandas as pd +from herbie import FastHerbie, Herbie + +from ...logging import redirect_stdout_to_logger from ._base import HRRRBaseDataset +logger = logging.getLogger(__name__) -class HRRRWindDataset(HRRRBaseDataset): - """ - HRRRWindDataset is a class that encaps a dataset from the HRRR + +class HRRRWindHourlyDataset(HRRRBaseDataset): + """HRRRWindDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. @@ -32,3 +40,53 @@ class HRRRWindDataset(HRRRBaseDataset): HRRR dataset. It allows users to specify the years and months of interest, as well as the variables they wish to download. """ + + weather_config = "wind" + product = "sfc" + + def _download_file(self, file: dict): + year, month = file["year"], file["month"] + + date_range = pd.date_range( + f"{year}-{month}-01", f"{year}-{month+1}-01", freq="h", inclusive="left" + ) + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + with redirect_stdout_to_logger(logger, logging.INFO): + logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") + fh.download(":[UV]GRD:[1,8]0 m") + + uv_10 = [] + uv_80 = [] + for hour in date_range: + h = Herbie( + hour, + model=self.module, + product=self.product, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + try: + uv_10.append(h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"})) + uv_80.append(h.xarray("[UV]GRD:80 m")) + except ValueError: + logger.warning(f"No data found for {hour}, skipping.") + + ds: xr.Dataset = xr.concat(uv_10 + uv_80, dim="heightAboveGround").rename( + {"latitude": "y", "longitude": "x"} + ) + ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + ds.to_netcdf(file["save_path"]) + + # NOTE: Flush temporary FastHerbie save directory to save space, since we no + # longer need the raw downloaded files + self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 20a77923..0605f292 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -27,6 +27,8 @@ class MERRA2BaseDataset(BaseDataset): """MERRA2BaseDataset is a class that encaps a dataset from the MERRA2 reanalysis dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. + + TODO: Support multi-file downloads. """ module = "merra2" diff --git a/src/geodata/logging.py b/src/geodata/logging.py index f1682fbb..78018a85 100644 --- a/src/geodata/logging.py +++ b/src/geodata/logging.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import contextlib import logging as _logging - color2num = { "gray": 30, "red": 31, @@ -54,6 +54,53 @@ def colorize( return f"\x1b[{attrs}m{string}\x1b[0m" +class StdoutToLoggerRedirect: + """A class to redirect stdout to a logger. + + Args: + logger: The logger to redirect stdout to. If not provided, a new logger + will be created. + level: The logging level to use when redirecting stdout. + Default is logging.INFO. + + Example: + >>> import logging + >>> with contextlib.redirect_stdout(StdoutToLoggerRedirect()): + ... print("Hello, world!") + """ + + def __init__( + self, logger: _logging.Logger | None = None, level: int = _logging.INFO + ): + self.logger = logger or _logging.getLogger(__name__) + self.level = level + + def write(self, msg: str): + if msg and not msg.isspace(): + self.logger.log(self.level, msg) + + def flush(self): + pass + + +@contextlib.contextmanager +def redirect_stdout_to_logger(logger=None, level=_logging.INFO): + """Context manager to redirect stdout to a logger. + + Args: + logger: The logger to redirect stdout to. If not provided, a new logger + will be created. + level: The logging level to use. Default is logging.INFO. + + Example: + >>> import logging + >>> with redirect_stdout_to_logger(): + ... print("Hello, world!") + """ + with contextlib.redirect_stdout(StdoutToLoggerRedirect(logger, level)): + yield + + class CustomFormatter(_logging.Formatter): # https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output @@ -83,3 +130,5 @@ def format(self, record): CustomFormatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ) logger.addHandler(ch) + +__all__ = ["logger", "redirect_stdout_to_logger"] From 32c69d10f8c6d45bd66d7ecd30aa7848a64bf544 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:56:41 -0800 Subject: [PATCH 07/54] feat: draft impl of wind_solar hourly dataset for HRRR --- src/geodata/datasets/_base.py | 2 +- src/geodata/datasets/hrrr/wind.py | 27 +++-- src/geodata/datasets/hrrr/wind_solar.py | 131 +++++++++++++++--------- 3 files changed, 101 insertions(+), 59 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 3d3ecce9..8cc7b852 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -126,7 +126,7 @@ def __init__( if not self.storage_root.exists(): logger.info( f"Storage directory for {self.__class__.__name__} does not exist, " - "creating now at {self.storage_root}" + f"creating now at {self.storage_root}" ) self.storage_root.mkdir(parents=True) diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py index d79ee1b4..2e550882 100644 --- a/src/geodata/datasets/hrrr/wind.py +++ b/src/geodata/datasets/hrrr/wind.py @@ -27,7 +27,7 @@ class HRRRWindHourlyDataset(HRRRBaseDataset): - """HRRRWindDataset is a class that encaps a dataset from the HRRR + """HRRRWindHourlyDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. @@ -75,16 +75,29 @@ def _download_file(self, file: dict): priority=self._priority, ) - try: - uv_10.append(h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"})) - uv_80.append(h.xarray("[UV]GRD:80 m")) - except ValueError: - logger.warning(f"No data found for {hour}, skipping.") + try: + uv_10.append( + h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"}) + ) + uv_80.append(h.xarray("[UV]GRD:80 m")) + except ValueError: + logger.warning(f"No data found for {hour}, skipping.") + + uv_10 = xr.concat(uv_10, dim="time") + uv_80 = xr.concat(uv_80, dim="time") - ds: xr.Dataset = xr.concat(uv_10 + uv_80, dim="heightAboveGround").rename( + ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround").rename( {"latitude": "y", "longitude": "x"} ) ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + + try: + del ds.attrs["search"] + del ds.attrs["local_grib"] + del ds.attrs["remote_grib"] + except KeyError: + pass + ds.to_netcdf(file["save_path"]) # NOTE: Flush temporary FastHerbie save directory to save space, since we no diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py index 563e8cb7..89729495 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -1,4 +1,4 @@ -# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,70 +13,99 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging - -from multiprocessing import cpu_count -from unittest.mock import patch - +import multiprocessing as mp +import xarray as xr import pandas as pd -import xarray as xr -from herbie import Herbie -from tqdm.auto import tqdm +from herbie import FastHerbie, Herbie +from ...logging import redirect_stdout_to_logger from ._base import HRRRBaseDataset logger = logging.getLogger(__name__) -def fake_print(*args, **kwargs): - pass - - -class HRRRWindSolarDataset(HRRRBaseDataset): - """HRRRWindSolarDataset is a class that encaps a dataset from the HRRR +class HRRRHourlyDataset(HRRRBaseDataset): + """HRRRHourlyDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. + + The HRRR dataset is a high-resolution weather forecast model that provides + hourly data for the United States. This dataset is useful for a variety of + applications, including renewable energy forecasting, weather prediction, + and climate research. + + This class provides a simple interface for downloading and processing the + HRRR dataset. It allows users to specify the years and months of interest, + as well as the variables they wish to download. """ weather_config = "wind_solar" - variables = ":[UV]GRD:[1,8]0 m" - - def download(self): - """Download the dataset from the HRRR dataset.""" - logger.info(f"Downloading {self.weather_config} dataset") - logger.info(self._herbie_save_dir.name) - - with patch("builtins.print", fake_print): - for file in tqdm(self.catalog, desc="Downloading Data", dynamic_ncols=True): - hours = pd.date_range( - f"{file['year']}-{file['month']}-01", - f"{file['year']}-{file['month']}-31", - freq="1h", + product = "sfc" + + def _download_file(self, file: dict): + year, month = file["year"], file["month"] + + date_range = pd.date_range( + f"{year}-{month}-01", f"{year}-{month+1}-01", freq="h", inclusive="left" + ) + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + with redirect_stdout_to_logger(logger, logging.INFO): + logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") + fh.download(":[UV]GRD:[1,8]0 m") + fh.download(":TMP:2 m") + + uv_10 = [] + uv_80 = [] + tmp_2 = [] + for hour in date_range: + h = Herbie( + hour, + model=self.module, + product=self.product, + save_dir=self._herbie_save_dir.name, + priority=self._priority, ) - dss = [] - for hour in hours: - h = Herbie( - hour, - fxx=0, - product="sfc", - model="hrrr", - priority=self._priority, - save_dir=self._herbie_save_dir.name, - max_threads=cpu_count() * 2, - ) - logger.info(f"Downloading {hour}") - dss.append( - xr.concat(h.xarray(self.variables), dim="heightAboveGround") + try: + uv_10.append(h.xarray("[UV]GRD:10 m", remove_grib=False)) + uv_80.append( + h.xarray("[UV]GRD:80 m", remove_grib=False).rename( + {"u": "u80", "v": "v80"} + ) ) - - ds = xr.concat(dss, dim="time").to_netcdf(file["save_path"]) - ds.close() - - logger.info(f"Downloaded {self.weather_config} dataset") - - @property - def downloaded(self): - pass + tmp_2.append(h.xarray("TMP:2 m", remove_grib=False)) + except ValueError: + logger.warning(f"No data found for {hour}, skipping.") + + uv_10: xr.Dataset = xr.concat(uv_10, dim="time") + uv_80: xr.Dataset = xr.concat(uv_80, dim="time") + tmp_2: xr.Dataset = xr.concat(tmp_2, dim="time") + + ds = xr.merge([uv_10, uv_80, tmp_2], compat="override").rename( + {"latitude": "y", "longitude": "x"} + ) + + try: + del ds["heightAboveGround"] + del ds.attrs["search"] + del ds.attrs["local_grib"] + del ds.attrs["remote_grib"] + except KeyError: + pass + + ds.to_netcdf(file["save_path"]) + + # NOTE: Flush temporary FastHerbie save directory to save space, since we no + # longer need the raw downloaded files + self._herbie_save_dir.cleanup() From e05be6e0ab2d4fdaeddaa06c4db77e23a5e5da79 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sat, 1 Feb 2025 20:31:04 -0800 Subject: [PATCH 08/54] feat: enhance HRRR dataset processing with postprocessing methods and daily frequency handling --- src/geodata/datasets/_base.py | 17 +++++++++-- src/geodata/datasets/hrrr/_base.py | 21 ++++++++++++++ src/geodata/datasets/hrrr/wind.py | 7 +++-- src/geodata/datasets/hrrr/wind_solar.py | 38 +++++++++++++++++-------- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 8cc7b852..41a21bd1 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -202,8 +202,21 @@ def download(self, force: bool = False): logger.info(f"Downloaded {self}") logger.info("Cleaning and renaming coordinates") + self.apply(self._dataset_postprocess) self.apply(self._rename_and_clean_coords) + def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): + """Method to postprocess the dataset after it has been downloaded. + This method should be implemented by subclasses to handle any + additional processing that is required for the dataset. + + Args: + ds: The dataset to postprocess. + **kwargs: Additional keyword arguments to pass to the function. + """ + + return ds + @apply def trim_variables( self, variables: Sequence[str] | None = None, **kwargs @@ -336,9 +349,7 @@ def _hourly_catalog(self): ) return catalog - def _rename_and_clean_coords( - ds: xr.Dataset | xr.DataArray, add_lon_lat: bool = True - ): + def _rename_and_clean_coords(ds: xr.Dataset, add_lon_lat: bool = True): """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index bba25071..7647d92f 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -15,8 +15,12 @@ import logging +import os.path as osp import tempfile +import herbie +import pandas as pd +import xarray as xr from .._base import BaseDataset @@ -31,6 +35,7 @@ class HRRRBaseDataset(BaseDataset): module = "hrrr" projection = "latlong" + frequency = "daily" _priority = ["google", "aws", "azure"] def _extra_setup(self, **kwargs): @@ -38,3 +43,19 @@ def _extra_setup(self, **kwargs): def __del__(self): self._herbie_save_dir.cleanup() + + def _preprocess_individual_herbie( + self, h: herbie.Herbie, search: str, hour: pd.DatetimeIndex + ): + tmp_ds = h.xarray(search, remove_grib=False) + tmp_ds.to_netcdf(osp.join(self._herbie_save_dir.name, f"{hour}_{search}.nc")) + tmp_ds.close() + + return osp.join(self._herbie_save_dir.name, f"{hour}_{search}.nc") + + def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): + # Because certain hours are missing, we need to reindex the dataset + # to include all hours in the range + + logger.debug("Reindexing dataset to include all hours in the range") + return ds.resample(time="1h").mean() diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py index 2e550882..87a13178 100644 --- a/src/geodata/datasets/hrrr/wind.py +++ b/src/geodata/datasets/hrrr/wind.py @@ -45,10 +45,13 @@ class HRRRWindHourlyDataset(HRRRBaseDataset): product = "sfc" def _download_file(self, file: dict): - year, month = file["year"], file["month"] + year, month, day = file["year"], file["month"], file["day"] date_range = pd.date_range( - f"{year}-{month}-01", f"{year}-{month+1}-01", freq="h", inclusive="left" + f"{year}-{month}-{day}", + f"{year}-{month}-{day+1}", + freq="h", + inclusive="left", ) fh = FastHerbie( diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py index 89729495..16e09937 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -15,9 +15,9 @@ import logging import multiprocessing as mp -import xarray as xr import pandas as pd +import xarray as xr from herbie import FastHerbie, Herbie from ...logging import redirect_stdout_to_logger @@ -45,10 +45,13 @@ class HRRRHourlyDataset(HRRRBaseDataset): product = "sfc" def _download_file(self, file: dict): - year, month = file["year"], file["month"] + year, month, day = file["year"], file["month"], file["day"] date_range = pd.date_range( - f"{year}-{month}-01", f"{year}-{month+1}-01", freq="h", inclusive="left" + f"{year}-{month}-{day}", + f"{year}-{month}-{day+1}" if day != 31 else f"{year}-{month+1}-01", + freq="h", + inclusive="left", ) fh = FastHerbie( @@ -61,7 +64,6 @@ def _download_file(self, file: dict): ) with redirect_stdout_to_logger(logger, logging.INFO): - logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") fh.download(":[UV]GRD:[1,8]0 m") fh.download(":TMP:2 m") @@ -78,19 +80,30 @@ def _download_file(self, file: dict): ) try: - uv_10.append(h.xarray("[UV]GRD:10 m", remove_grib=False)) + uv_10.append( + self._preprocess_individual_herbie(h, "[UV]GRD:10 m", hour) + ) uv_80.append( - h.xarray("[UV]GRD:80 m", remove_grib=False).rename( - {"u": "u80", "v": "v80"} - ) + self._preprocess_individual_herbie(h, "[UV]GRD:80 m", hour) ) - tmp_2.append(h.xarray("TMP:2 m", remove_grib=False)) + tmp_2.append( + self._preprocess_individual_herbie(h, ":TMP:2 m", hour) + ) + except ValueError: logger.warning(f"No data found for {hour}, skipping.") - uv_10: xr.Dataset = xr.concat(uv_10, dim="time") - uv_80: xr.Dataset = xr.concat(uv_80, dim="time") - tmp_2: xr.Dataset = xr.concat(tmp_2, dim="time") + # First concat to daily files + + uv_10: xr.Dataset = xr.open_mfdataset( + uv_10, concat_dim="time", chunks="auto", combine="nested" + ) + uv_80: xr.Dataset = xr.open_mfdataset( + uv_80, concat_dim="time", chunks="auto", combine="nested" + ) + tmp_2: xr.Dataset = xr.open_mfdataset( + tmp_2, concat_dim="time", chunks="auto", combine="nested" + ) ds = xr.merge([uv_10, uv_80, tmp_2], compat="override").rename( {"latitude": "y", "longitude": "x"} @@ -101,6 +114,7 @@ def _download_file(self, file: dict): del ds.attrs["search"] del ds.attrs["local_grib"] del ds.attrs["remote_grib"] + del ds.coords["gribfile_projection"] except KeyError: pass From bf5187c5f0b18b598efc1566287783b264e06c21 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:23:53 -0800 Subject: [PATCH 09/54] feat: Use AtomicDataset to represent individual xarray files rather than a dict --- src/geodata/datasets/_base.py | 183 ++++++++++++++---- .../datasets/era5/hourly/wind_solar.py | 9 +- .../datasets/era5/monthly/wind_solar.py | 9 +- src/geodata/datasets/hrrr/_base.py | 9 +- src/geodata/datasets/hrrr/wind.py | 13 +- src/geodata/datasets/hrrr/wind_3d.py | 98 ++++++++++ src/geodata/datasets/hrrr/wind_solar.py | 11 +- src/geodata/datasets/merra2/_base.py | 14 +- 8 files changed, 279 insertions(+), 67 deletions(-) create mode 100644 src/geodata/datasets/hrrr/wind_3d.py diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 41a21bd1..2db51d04 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -14,6 +14,8 @@ # along with this program. If not, see . import abc +import dataclasses +import hashlib import itertools import logging from collections.abc import Sequence @@ -29,6 +31,82 @@ logger = logging.getLogger(__name__) +@dataclasses.dataclass +class AtomicDataset: + """AtomicDataset is a class that encapsulates an individual xarray file that was + downloaded. It provides a streamlined workflow for downloading, preprocessing, + and integrity checking of these datasets. + """ + + year: int + month: int + dataset: "BaseDataset" + day: int | None = None + file_hash: str | None = None + url: str | None = None + spinup: bool | None = None + + def __post_init__(self): + if not isinstance(self.dataset, BaseDataset): + raise ValueError("dataset must be an instance of BaseDataset") + + @property + def path(self): + """The path where the file should be saved""" + if self.day is None: + return self.dataset.storage_root / str(self.year) / f"{self.month:02d}.nc" + else: + return ( + self.dataset.storage_root + / str(self.year) + / f"{self.month:02d}" + / f"{self.day:02d}.nc" + ) + + def check(self, integrity: bool = True): + """Check the presence of the file and its integrity. + + Args: + integrity: A boolean flag indicating whether to check the integrity + of the file. If True, the file will be checked against its hash. + If False, only the presence of the file will be checked. + + Returns: + True if the file is present and its integrity is intact, False otherwise. + """ + + if not self.path.exists(): + logger.debug(f"{self.path} does not exist") + return False + + if not integrity: + return True + + # In case the file just got downloaded + if self.file_hash is None: + self.file_hash = self._compute_hash() + return True + + if self.file_hash != self._compute_hash(): + logger.warning(f"{self.path} is corrupted") + return False + + return True + + def _compute_hash(self): + """Compute the hash of the file. + + Returns: + The hash of the file. + """ + + hash_func = hashlib.sha256() + with open(self.path, "rb") as f: + while chunk := f.read(8192): # Read file in chunks + hash_func.update(chunk) + return hash_func.hexdigest() + + class BaseDataset(abc.ABC): """Dataset is a class that encapsulates any datasets natively supported by geodata. It provides a streamlined workflow for downloading, preprocessing, @@ -130,6 +208,7 @@ def __init__( ) self.storage_root.mkdir(parents=True) + self._extra_kwargs = kwargs self._extra_setup(**kwargs) def _extra_setup(self, **kwargs): @@ -149,12 +228,35 @@ def apply(func: callable, *args, **kwargs): def wrapper(self, *args, **kwargs): for file in self.catalog: - with xr.open_dataset(file["save_path"], chunks="auto") as ds: + with xr.open_dataset(file.path, chunks="auto") as ds: ds = func(ds, *args, **kwargs) - ds.to_netcdf(file["save_path"]) + ds.to_netcdf(file.path) return wrapper + def _generate_manifest(self): + """Generate a manifest file for the dataset. This file contains + metadata about the dataset, including the file paths and their + integrity checks. + + TODO! This method is not ready yet! + + Returns: + A list of dictionaries containing the metadata of each file in the + dataset. + """ + + manifest = [] + for file in self.catalog: + manifest.append( + { + "path": str(file["save_path"]), + "integrity": file["downloaded"], + } + ) + + return manifest + @property def downloaded(self): """A boolean flag indicating whether the dataset has been prepared @@ -166,7 +268,7 @@ def downloaded(self): a more comprehensive check is required. """ - return all((file["downloaded"] for file in self.catalog)) + return all((file.check() for file in self.catalog)) @abc.abstractmethod def _download_file(self, file: dict): @@ -262,6 +364,20 @@ def projection(self): def extra_repr(self): return "" + @property + def testing(self): + """A boolean flag indicating whether the dataset is being used for + testing. Under this mode, only the first few days or months of the dataset + will be downloaded (depending on the granularity). This is useful for + testing the dataset without downloading the entire dataset. + """ + + if "testing" not in self._extra_kwargs: + return False + if not isinstance(self._extra_kwargs["testing"], bool): + raise ValueError("testing must be a boolean flag") + return self._extra_kwargs["testing"] + @property def submodule(self): """The submodule of the dataset. This can be defined by the dataset @@ -271,7 +387,7 @@ def submodule(self): return getattr(self, "weather_config", self.__class__.__name__) @property - def catalog(self): + def catalog(self) -> list["AtomicDataset"]: """A generator that yields all the files that need to be downloaded. Each iteration should return a dictionary with the following keys: - year: the year of the file @@ -293,9 +409,6 @@ def catalog(self): f"Invalid frequency {self.frequency} defined for this dataset." ) - for file in cat: - file["downloaded"] = file["save_path"].exists() - return cat def _monthly_catalog(self): @@ -305,12 +418,11 @@ def _monthly_catalog(self): range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), ): - save_path = self.storage_root / f"{year}_{month:02d}.nc" - catalog.append({"year": year, "month": month, "save_path": save_path}) + catalog.append(AtomicDataset(year, month, dataset=self)) return catalog - def _daily_catalog(self): + def _daily_catalog(self) -> list["AtomicDataset"]: catalog = [] for year, month in itertools.product( @@ -318,36 +430,33 @@ def _daily_catalog(self): range(self.months.start, self.months.stop + 1), ): for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): - save_path = self.storage_root / f"{year}_{month:02d}_{day:02d}.nc" - catalog.append( - {"year": year, "month": month, "day": day, "save_path": save_path} - ) + catalog.append(AtomicDataset(year, month, day, dataset=self)) return catalog - def _hourly_catalog(self): - catalog = [] - - for year, month in itertools.product( - range(self.years.start, self.years.stop + 1), - range(self.months.start, self.months.stop + 1), - ): - for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): - for hour in range(24): - save_path = ( - self.storage_root - / f"{year}_{month:02d}_{day:02d}_{hour:02d}.nc" - ) - catalog.append( - { - "year": year, - "month": month, - "day": day, - "hour": hour, - "save_path": save_path, - } - ) - return catalog + # def _hourly_catalog(self): + # catalog = [] + + # for year, month in itertools.product( + # range(self.years.start, self.years.stop + 1), + # range(self.months.start, self.months.stop + 1), + # ): + # for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): + # for hour in range(24): + # save_path = ( + # self.storage_root + # / f"{year}_{month:02d}_{day:02d}_{hour:02d}.nc" + # ) + # catalog.append( + # { + # "year": year, + # "month": month, + # "day": day, + # "hour": hour, + # "save_path": save_path, + # } + # ) + # return catalog def _rename_and_clean_coords(ds: xr.Dataset, add_lon_lat: bool = True): """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py index 11da5f94..377f6b7e 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -22,6 +22,7 @@ import xarray as xr +from ..._base import AtomicDataset from .._base import ERA5BaseDataset logger = logging.getLogger(__name__) @@ -73,10 +74,10 @@ class ERA5WindSolarHourlyDataset(ERA5BaseDataset): product = "reanalysis-era5-single-levels" product_type = "reanalysis" - def _download_file(self, file: dict): - year: int = file["year"] - month: int = file["month"] - save_path: Path = file["save_path"] + def _download_file(self, file: AtomicDataset): + year: int = file.year + month: int = file.month + save_path: Path = file.path full_request = { "product_type": self.product_type, diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py index fbff555a..9c1d4172 100644 --- a/src/geodata/datasets/era5/monthly/wind_solar.py +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -22,6 +22,7 @@ import xarray as xr +from ..._base import AtomicDataset from ..hourly.wind_solar import ERA5WindSolarHourlyDataset logger = logging.getLogger(__name__) @@ -55,10 +56,10 @@ class ERA5WindSolarMonthlyDataset(ERA5WindSolarHourlyDataset): weather_config = "wind_solar_monthly" - def _download_file(self, file: dict): - year: int = file["year"] - month: int = file["month"] - save_path: Path = file["save_path"] + def _download_file(self, file: AtomicDataset): + year: int = file.year + month: int = file.month + save_path: Path = file.path full_request = { "product_type": self.product_type, diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index 7647d92f..6898d96d 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -58,4 +58,11 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): # to include all hours in the range logger.debug("Reindexing dataset to include all hours in the range") - return ds.resample(time="1h").mean() + ds = ds.resample(time="1h").mean() + + # NOTE: For some reasons, HRRR's longitude is in the range of [0, 360] + # instead of [-180, 180]. We need to put it back to [-180, 180]. + logger.debug("Fixing longitude range") + ds["longitude"] = (ds["longitude"] % 360 + 540) % 360 - 180 + + return ds diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py index 87a13178..6cefab69 100644 --- a/src/geodata/datasets/hrrr/wind.py +++ b/src/geodata/datasets/hrrr/wind.py @@ -15,12 +15,13 @@ import logging import multiprocessing as mp -import xarray as xr import pandas as pd +import xarray as xr from herbie import FastHerbie, Herbie from ...logging import redirect_stdout_to_logger +from .._base import AtomicDataset from ._base import HRRRBaseDataset logger = logging.getLogger(__name__) @@ -44,8 +45,8 @@ class HRRRWindHourlyDataset(HRRRBaseDataset): weather_config = "wind" product = "sfc" - def _download_file(self, file: dict): - year, month, day = file["year"], file["month"], file["day"] + def _download_file(self, file: AtomicDataset): + year, month, day = file.year, file.month, file.day date_range = pd.date_range( f"{year}-{month}-{day}", @@ -89,9 +90,7 @@ def _download_file(self, file: dict): uv_10 = xr.concat(uv_10, dim="time") uv_80 = xr.concat(uv_80, dim="time") - ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround").rename( - {"latitude": "y", "longitude": "x"} - ) + ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 try: @@ -101,7 +100,7 @@ def _download_file(self, file: dict): except KeyError: pass - ds.to_netcdf(file["save_path"]) + ds.to_netcdf(file.path) # NOTE: Flush temporary FastHerbie save directory to save space, since we no # longer need the raw downloaded files diff --git a/src/geodata/datasets/hrrr/wind_3d.py b/src/geodata/datasets/hrrr/wind_3d.py new file mode 100644 index 00000000..5cab0261 --- /dev/null +++ b/src/geodata/datasets/hrrr/wind_3d.py @@ -0,0 +1,98 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import multiprocessing as mp + +import pandas as pd +import xarray as xr +from herbie import FastHerbie, Herbie + +from ...logging import redirect_stdout_to_logger +from .._base import AtomicDataset +from ._base import HRRRBaseDataset + +logger = logging.getLogger(__name__) + + +class HRRR3DWindHourlyDataset(HRRRBaseDataset): + """HRRR3DWindHourlyDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "wind" + product = "nat" # Use "nat" product for 3D data + + def _download_file(self, file: AtomicDataset): + year, month, day = file.year, file.month, file.day + + date_range = pd.date_range( + f"{year}-{month}-{day}", + f"{year}-{month}-{day+1}", + freq="h", + inclusive="left", + ) + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + with redirect_stdout_to_logger(logger, logging.INFO): + logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") + fh.download(":[UV]GRD:[1,8]0 m") + + uv_10 = [] + uv_80 = [] + for hour in date_range: + h = Herbie( + hour, + model=self.module, + product=self.product, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + try: + uv_10.append( + h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"}) + ) + uv_80.append(h.xarray("[UV]GRD:80 m")) + except ValueError: + logger.warning(f"No data found for {hour}, skipping.") + + uv_10 = xr.concat(uv_10, dim="time") + uv_80 = xr.concat(uv_80, dim="time") + + ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") + ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + + try: + del ds.attrs["search"] + del ds.attrs["local_grib"] + del ds.attrs["remote_grib"] + except KeyError: + pass + + ds.to_netcdf(file.path) + + # NOTE: Flush temporary FastHerbie save directory to save space, since we no + # longer need the raw downloaded files + self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py index 16e09937..d25f76a1 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -21,6 +21,7 @@ from herbie import FastHerbie, Herbie from ...logging import redirect_stdout_to_logger +from .._base import AtomicDataset from ._base import HRRRBaseDataset logger = logging.getLogger(__name__) @@ -44,8 +45,8 @@ class HRRRHourlyDataset(HRRRBaseDataset): weather_config = "wind_solar" product = "sfc" - def _download_file(self, file: dict): - year, month, day = file["year"], file["month"], file["day"] + def _download_file(self, file: AtomicDataset): + year, month, day = file.year, file.month, file.day date_range = pd.date_range( f"{year}-{month}-{day}", @@ -105,9 +106,7 @@ def _download_file(self, file: dict): tmp_2, concat_dim="time", chunks="auto", combine="nested" ) - ds = xr.merge([uv_10, uv_80, tmp_2], compat="override").rename( - {"latitude": "y", "longitude": "x"} - ) + ds = xr.merge([uv_10, uv_80, tmp_2], compat="override") try: del ds["heightAboveGround"] @@ -118,7 +117,7 @@ def _download_file(self, file: dict): except KeyError: pass - ds.to_netcdf(file["save_path"]) + ds.to_netcdf(file.path) # NOTE: Flush temporary FastHerbie save directory to save space, since we no # longer need the raw downloaded files diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 0605f292..6e33981f 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -13,14 +13,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from pathlib import Path import numpy as np import requests import xarray as xr from ...types import CoordRange -from .._base import BaseDataset +from .._base import AtomicDataset, BaseDataset class MERRA2BaseDataset(BaseDataset): @@ -36,17 +35,16 @@ class MERRA2BaseDataset(BaseDataset): frequency = "daily" url_template = "" - def _download_file(self, file: dict): + def _download_file(self, file: AtomicDataset): assert "url" in file, "URL is required to download the file" - url: str = file["url"] - path: Path = file["save_path"] + url: str = file.url # Download the file with requests.get(url, stream=True) as r: r.raise_for_status() - with open(path, "wb") as f: + with open(file.path, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) @@ -125,7 +123,7 @@ def _daily_catalog(self): catalog = super()._daily_catalog() for file in catalog: - file["spinup"] = self.spinup_year(file["year"], file["month"]) - file["url"] = self.url_template.format(**file) + file.spinup = self.spinup_year(file.year, file.month) + file.url = self.url_template.format(**vars(file)) return catalog From f4d96cc001109ea3d9b4cad4a04dfe1440c336e7 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:24:09 -0800 Subject: [PATCH 10/54] feat: update HRRR dataset processing to include WRF data and improve coordinate handling --- pyproject.toml | 3 ++ src/geodata/datasets/_base.py | 57 ++++++++++------------ src/geodata/datasets/hrrr/_base.py | 2 +- src/geodata/datasets/hrrr/wind_solar.py | 14 ++++-- uv.lock | 64 +++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 35 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1bc5f6f5..44a93941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,9 @@ docs = [ "sphinx-book-theme>=1.1.3", "sphinx-autoapi==3.3.2" ] +accelerate = [ + "numba>=0.61.0", +] [tool.uv] dev-dependencies = [ diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 2db51d04..225b7789 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -38,9 +38,9 @@ class AtomicDataset: and integrity checking of these datasets. """ + dataset: "BaseDataset" year: int month: int - dataset: "BaseDataset" day: int | None = None file_hash: str | None = None url: str | None = None @@ -216,24 +216,6 @@ def _extra_setup(self, **kwargs): that is required for the dataset. """ - @staticmethod - def apply(func: callable, *args, **kwargs): - """Method to apply a function to each file in the dataset. - - Args: - func: A function that takes an xarray.Dataset as its first argument. - *args: Additional arguments to pass to the function. - **kwargs: Additional keyword arguments to pass to the function. - """ - - def wrapper(self, *args, **kwargs): - for file in self.catalog: - with xr.open_dataset(file.path, chunks="auto") as ds: - ds = func(ds, *args, **kwargs) - ds.to_netcdf(file.path) - - return wrapper - def _generate_manifest(self): """Generate a manifest file for the dataset. This file contains metadata about the dataset, including the file paths and their @@ -299,13 +281,22 @@ def download(self, force: bool = False): return for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): + # We first must ensure the directory exists + file.path.parent.mkdir(parents=True, exist_ok=True) + self._download_file(file) logger.info(f"Downloaded {self}") logger.info("Cleaning and renaming coordinates") - self.apply(self._dataset_postprocess) - self.apply(self._rename_and_clean_coords) + # Post-process the dataset + for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): + if file.check(): + ds = xr.open_dataset(file.path) + ds = self._rename_and_clean_coords(ds) + ds = self._dataset_postprocess(ds) + ds.to_netcdf(file.path) + ds.close() def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): """Method to postprocess the dataset after it has been downloaded. @@ -319,7 +310,6 @@ def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): return ds - @apply def trim_variables( self, variables: Sequence[str] | None = None, **kwargs ) -> xr.Dataset | xr.DataArray: @@ -418,7 +408,7 @@ def _monthly_catalog(self): range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), ): - catalog.append(AtomicDataset(year, month, dataset=self)) + catalog.append(AtomicDataset(self, year, month)) return catalog @@ -429,8 +419,13 @@ def _daily_catalog(self) -> list["AtomicDataset"]: range(self.years.start, self.years.stop + 1), range(self.months.start, self.months.stop + 1), ): - for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): - catalog.append(AtomicDataset(year, month, day, dataset=self)) + for day in range( + 1, + pd.Timestamp(f"{year}-{month}-1").days_in_month + 1 + if not self.testing + else 3, + ): + catalog.append(AtomicDataset(self, year, month, day)) return catalog @@ -458,27 +453,27 @@ def _daily_catalog(self) -> list["AtomicDataset"]: # ) # return catalog - def _rename_and_clean_coords(ds: xr.Dataset, add_lon_lat: bool = True): + def _rename_and_clean_coords(self, ds: xr.Dataset, add_lon_lat: bool = False): """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. Args: ds (xarray.Dataset): Dataset to rename - add_lon_lat (bool, optional): Add lon/lat columns. Defaults to True. + add_lon_lat (bool, optional): Add lon/lat columns. Defaults to False. Returns: xarray.Dataset: Dataset with renamed coordinates """ # Rename latitude / lat -> y, longitude / lon -> x - if "latitude" in list(ds.coords): + if "latitude" in ds.coords: ds = ds.rename({"latitude": "y"}) - if "longitude" in list(ds.coords): + if "longitude" in ds.coords: ds = ds.rename({"longitude": "x"}) - if "lat" in list(ds.coords): + if "lat" in ds.coords: ds = ds.rename({"lat": "y"}) - if "lon" in list(ds.coords): + if "lon" in ds.coords: ds = ds.rename({"lon": "x"}) if add_lon_lat: diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index 6898d96d..2c81127e 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -63,6 +63,6 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): # NOTE: For some reasons, HRRR's longitude is in the range of [0, 360] # instead of [-180, 180]. We need to put it back to [-180, 180]. logger.debug("Fixing longitude range") - ds["longitude"] = (ds["longitude"] % 360 + 540) % 360 - 180 + ds["x"] = (ds["x"] % 360 + 540) % 360 - 180 return ds diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py index d25f76a1..881fbc2d 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -50,7 +50,7 @@ def _download_file(self, file: AtomicDataset): date_range = pd.date_range( f"{year}-{month}-{day}", - f"{year}-{month}-{day+1}" if day != 31 else f"{year}-{month+1}-01", + f"{year}-{month}-{day+1}", freq="h", inclusive="left", ) @@ -71,6 +71,8 @@ def _download_file(self, file: AtomicDataset): uv_10 = [] uv_80 = [] tmp_2 = [] + wrfs = [] + for hour in date_range: h = Herbie( hour, @@ -90,6 +92,9 @@ def _download_file(self, file: AtomicDataset): tmp_2.append( self._preprocess_individual_herbie(h, ":TMP:2 m", hour) ) + wrfs.append( + self._preprocess_individual_herbie(h, ":..WRF:surface", hour) + ) except ValueError: logger.warning(f"No data found for {hour}, skipping.") @@ -101,12 +106,15 @@ def _download_file(self, file: AtomicDataset): ) uv_80: xr.Dataset = xr.open_mfdataset( uv_80, concat_dim="time", chunks="auto", combine="nested" - ) + ).rename({"u": "u80", "v": "v80"}) tmp_2: xr.Dataset = xr.open_mfdataset( tmp_2, concat_dim="time", chunks="auto", combine="nested" ) + wrfs: xr.Dataset = xr.open_mfdataset( + wrfs, concat_dim="time", chunks="auto", combine="nested" + ) - ds = xr.merge([uv_10, uv_80, tmp_2], compat="override") + ds = xr.merge([uv_10, uv_80, tmp_2, wrfs], compat="override") try: del ds["heightAboveGround"] diff --git a/uv.lock b/uv.lock index 9e0125dc..dcc5387e 100644 --- a/uv.lock +++ b/uv.lock @@ -817,6 +817,9 @@ dependencies = [ ] [package.optional-dependencies] +accelerate = [ + { name = "numba" }, +] docs = [ { name = "myst-nb" }, { name = "sphinx" }, @@ -851,6 +854,7 @@ requires-dist = [ { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "netcdf4", specifier = ">=1.7.1.post2" }, { name = "notebook", marker = "extra == 'notebook'", specifier = ">=7.2.2" }, + { name = "numba", marker = "extra == 'accelerate'", specifier = ">=0.61.0" }, { name = "numexpr", specifier = "==2.10.1" }, { name = "numpy", specifier = "<2" }, { name = "pandas", specifier = ">=2.2.3" }, @@ -1450,6 +1454,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/8b/d7497df4a1cae9367adf21665dd1f896c2a7aeb8769ad77b662c5e2bcce7/kiwisolver-1.4.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:eb542fe7933aa09d8d8f9d9097ef37532a7df6497819d16efe4359890a2f417a", size = 55715 }, ] +[[package]] +name = "llvmlite" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/6a/95a3d3610d5c75293d5dbbb2a76480d5d4eeba641557b69fe90af6c5b84e/llvmlite-0.44.0.tar.gz", hash = "sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4", size = 171880 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/75/d4863ddfd8ab5f6e70f4504cf8cc37f4e986ec6910f4ef8502bb7d3c1c71/llvmlite-0.44.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9fbadbfba8422123bab5535b293da1cf72f9f478a65645ecd73e781f962ca614", size = 28132306 }, + { url = "https://files.pythonhosted.org/packages/37/d9/6e8943e1515d2f1003e8278819ec03e4e653e2eeb71e4d00de6cfe59424e/llvmlite-0.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cccf8eb28f24840f2689fb1a45f9c0f7e582dd24e088dcf96e424834af11f791", size = 26201096 }, + { url = "https://files.pythonhosted.org/packages/aa/46/8ffbc114def88cc698906bf5acab54ca9fdf9214fe04aed0e71731fb3688/llvmlite-0.44.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7202b678cdf904823c764ee0fe2dfe38a76981f4c1e51715b4cb5abb6cf1d9e8", size = 42361859 }, + { url = "https://files.pythonhosted.org/packages/30/1c/9366b29ab050a726af13ebaae8d0dff00c3c58562261c79c635ad4f5eb71/llvmlite-0.44.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40526fb5e313d7b96bda4cbb2c85cd5374e04d80732dd36a282d72a560bb6408", size = 41184199 }, + { url = "https://files.pythonhosted.org/packages/69/07/35e7c594b021ecb1938540f5bce543ddd8713cff97f71d81f021221edc1b/llvmlite-0.44.0-cp310-cp310-win_amd64.whl", hash = "sha256:41e3839150db4330e1b2716c0be3b5c4672525b4c9005e17c7597f835f351ce2", size = 30332381 }, + { url = "https://files.pythonhosted.org/packages/b5/e2/86b245397052386595ad726f9742e5223d7aea999b18c518a50e96c3aca4/llvmlite-0.44.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3", size = 28132305 }, + { url = "https://files.pythonhosted.org/packages/ff/ec/506902dc6870249fbe2466d9cf66d531265d0f3a1157213c8f986250c033/llvmlite-0.44.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ace564d9fa44bb91eb6e6d8e7754977783c68e90a471ea7ce913bff30bd62427", size = 26201090 }, + { url = "https://files.pythonhosted.org/packages/99/fe/d030f1849ebb1f394bb3f7adad5e729b634fb100515594aca25c354ffc62/llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5d22c3bfc842668168a786af4205ec8e3ad29fb1bc03fd11fd48460d0df64c1", size = 42361858 }, + { url = "https://files.pythonhosted.org/packages/d7/7a/ce6174664b9077fc673d172e4c888cb0b128e707e306bc33fff8c2035f0d/llvmlite-0.44.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610", size = 41184200 }, + { url = "https://files.pythonhosted.org/packages/5f/c6/258801143975a6d09a373f2641237992496e15567b907a4d401839d671b8/llvmlite-0.44.0-cp311-cp311-win_amd64.whl", hash = "sha256:d8489634d43c20cd0ad71330dde1d5bc7b9966937a263ff1ec1cebb90dc50955", size = 30331193 }, + { url = "https://files.pythonhosted.org/packages/15/86/e3c3195b92e6e492458f16d233e58a1a812aa2bfbef9bdd0fbafcec85c60/llvmlite-0.44.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:1d671a56acf725bf1b531d5ef76b86660a5ab8ef19bb6a46064a705c6ca80aad", size = 28132297 }, + { url = "https://files.pythonhosted.org/packages/d6/53/373b6b8be67b9221d12b24125fd0ec56b1078b660eeae266ec388a6ac9a0/llvmlite-0.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f79a728e0435493611c9f405168682bb75ffd1fbe6fc360733b850c80a026db", size = 26201105 }, + { url = "https://files.pythonhosted.org/packages/cb/da/8341fd3056419441286c8e26bf436923021005ece0bff5f41906476ae514/llvmlite-0.44.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0143a5ef336da14deaa8ec26c5449ad5b6a2b564df82fcef4be040b9cacfea9", size = 42361901 }, + { url = "https://files.pythonhosted.org/packages/53/ad/d79349dc07b8a395a99153d7ce8b01d6fcdc9f8231355a5df55ded649b61/llvmlite-0.44.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d752f89e31b66db6f8da06df8b39f9b91e78c5feea1bf9e8c1fba1d1c24c065d", size = 41184247 }, + { url = "https://files.pythonhosted.org/packages/e2/3b/a9a17366af80127bd09decbe2a54d8974b6d8b274b39bf47fbaedeec6307/llvmlite-0.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:eae7e2d4ca8f88f89d315b48c6b741dcb925d6a1042da694aa16ab3dd4cbd3a1", size = 30332380 }, + { url = "https://files.pythonhosted.org/packages/89/24/4c0ca705a717514c2092b18476e7a12c74d34d875e05e4d742618ebbf449/llvmlite-0.44.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:319bddd44e5f71ae2689859b7203080716448a3cd1128fb144fe5c055219d516", size = 28132306 }, + { url = "https://files.pythonhosted.org/packages/01/cf/1dd5a60ba6aee7122ab9243fd614abcf22f36b0437cbbe1ccf1e3391461c/llvmlite-0.44.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c58867118bad04a0bb22a2e0068c693719658105e40009ffe95c7000fcde88e", size = 26201090 }, + { url = "https://files.pythonhosted.org/packages/d2/1b/656f5a357de7135a3777bd735cc7c9b8f23b4d37465505bd0eaf4be9befe/llvmlite-0.44.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46224058b13c96af1365290bdfebe9a6264ae62fb79b2b55693deed11657a8bf", size = 42361904 }, + { url = "https://files.pythonhosted.org/packages/d8/e1/12c5f20cb9168fb3464a34310411d5ad86e4163c8ff2d14a2b57e5cc6bac/llvmlite-0.44.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0097052c32bf721a4efc03bd109d335dfa57d9bffb3d4c24cc680711b8b4fc", size = 41184245 }, + { url = "https://files.pythonhosted.org/packages/d0/81/e66fc86539293282fd9cb7c9417438e897f369e79ffb62e1ae5e5154d4dd/llvmlite-0.44.0-cp313-cp313-win_amd64.whl", hash = "sha256:2fb7c4f2fb86cbae6dca3db9ab203eeea0e22d73b99bc2341cdf9de93612e930", size = 30331193 }, +] + [[package]] name = "locket" version = "1.0.0" @@ -1779,6 +1811,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307 }, ] +[[package]] +name = "numba" +version = "0.61.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/88/c13a935f200fda51384411e49840a8e7f70c9cb1ee8d809dd0f2477cf7ef/numba-0.61.0.tar.gz", hash = "sha256:888d2e89b8160899e19591467e8fdd4970e07606e1fbc248f239c89818d5f925", size = 2816484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/97/8568a025b9ab8b4d53491e70d4206d5f3fc71fbe94f3097058e01ad8e7ff/numba-0.61.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9cab9783a700fa428b1a54d65295122bc03b3de1d01fb819a6b9dbbddfdb8c43", size = 2769008 }, + { url = "https://files.pythonhosted.org/packages/8c/ab/a88c20755f66543ee01c85c98b866595b92e1bd0ed80565a4889e22929a8/numba-0.61.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:46c5ae094fb3706f5adf9021bfb7fc11e44818d61afee695cdee4eadfed45e98", size = 2771815 }, + { url = "https://files.pythonhosted.org/packages/ae/f4/b357913089ecec1a9ddc6adc04090396928f36a484a5ab9e71b24ddba4cd/numba-0.61.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6fb74e81aa78a2303e30593d8331327dfc0d2522b5db05ac967556a26db3ef87", size = 3820233 }, + { url = "https://files.pythonhosted.org/packages/ea/60/0e21bcf3baaf10e39d48cd224618e46a6b75d3394f465c37ce57bf98cbfa/numba-0.61.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ebbd4827091384ab8c4615ba1b3ca8bc639a3a000157d9c37ba85d34cd0da1b", size = 3514707 }, + { url = "https://files.pythonhosted.org/packages/a0/08/45c136ab59e6b11e61ce15a0d17ef03fd89eaccb0db05ad67912aaf5218a/numba-0.61.0-cp310-cp310-win_amd64.whl", hash = "sha256:43aa4d7d10c542d3c78106b8481e0cbaaec788c39ee8e3d7901682748ffdf0b4", size = 2827753 }, + { url = "https://files.pythonhosted.org/packages/63/8f/f983a7c859ccad73d3cc3f86fbba94f16e137cd1ee464631d61b624363b2/numba-0.61.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:bf64c2d0f3d161af603de3825172fb83c2600bcb1d53ae8ea568d4c53ba6ac08", size = 2768960 }, + { url = "https://files.pythonhosted.org/packages/be/1b/c33dc847d475d5b647b4ad5aefc38df7a72283763f4cda47745050375a81/numba-0.61.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de5aa7904741425f28e1028b85850b31f0a245e9eb4f7c38507fb893283a066c", size = 2771862 }, + { url = "https://files.pythonhosted.org/packages/14/91/18b9f64b34ff318a14d072251480547f89ebfb864b2b7168e5dc5f64f502/numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21c2fe25019267a608e2710a6a947f557486b4b0478b02e45a81cf606a05a7d4", size = 3825411 }, + { url = "https://files.pythonhosted.org/packages/f2/97/1a38030c2a331e273ace1de2b61988e33d80878fda8a5eedee0cd78399d3/numba-0.61.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:74250b26ed6a1428763e774dc5b2d4e70d93f73795635b5412b8346a4d054574", size = 3519604 }, + { url = "https://files.pythonhosted.org/packages/df/a7/56f547de8fc197963f238fd62beb5f1d2cace047602d0577956bf6840970/numba-0.61.0-cp311-cp311-win_amd64.whl", hash = "sha256:b72bbc8708e98b3741ad0c63f9929c47b623cc4ee86e17030a4f3e301e8401ac", size = 2827642 }, + { url = "https://files.pythonhosted.org/packages/63/c9/c61881e7f2e253e745209f078bbd428ce23b6cf901f7d93afe166720ff95/numba-0.61.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:152146ecdbb8d8176f294e9f755411e6f270103a11c3ff50cecc413f794e52c8", size = 2769758 }, + { url = "https://files.pythonhosted.org/packages/e1/28/ddec0147a4933f86ceaca580aa9bb767d5632ecdb1ece6cfb3eab4ac78e5/numba-0.61.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5cafa6095716fcb081618c28a8d27bf7c001e09696f595b41836dec114be2905", size = 2772445 }, + { url = "https://files.pythonhosted.org/packages/18/74/6a9f0e6c76c088f8a6aa702eab31734068061dca5cc0f34e8bc1eb447de1/numba-0.61.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ffe9fe373ed30638d6e20a0269f817b2c75d447141f55a675bfcf2d1fe2e87fb", size = 3882115 }, + { url = "https://files.pythonhosted.org/packages/53/68/d7c31e53f08e6b4669c9b5a3cd7c5fb9097220c5ef388bc099ca8ab9749f/numba-0.61.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9f25f7fef0206d55c1cfb796ad833cbbc044e2884751e56e798351280038484c", size = 3573296 }, + { url = "https://files.pythonhosted.org/packages/94/4f/8357a99a14f331b865a42cb4756ae37da85599b9c95e01277ea10361e91a/numba-0.61.0-cp312-cp312-win_amd64.whl", hash = "sha256:550d389573bc3b895e1ccb18289feea11d937011de4d278b09dc7ed585d1cdcb", size = 2828077 }, + { url = "https://files.pythonhosted.org/packages/3b/54/71fba18e4af5619f1ea8175ee92e82dd8e220bd6feb8c0153c6b814c8a60/numba-0.61.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:b96fafbdcf6f69b69855273e988696aae4974115a815f6818fef4af7afa1f6b8", size = 2768024 }, + { url = "https://files.pythonhosted.org/packages/39/76/2448b43d08e904aad1b1b9cd12835b19411e84a81aa9192f83642a5e0afd/numba-0.61.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f6c452dca1de8e60e593f7066df052dd8da09b243566ecd26d2b796e5d3087d", size = 2769541 }, + { url = "https://files.pythonhosted.org/packages/32/8f/4bb2374247ab988c9eac587b304b2947a36d605b9bb9ba4bf06e955c17d3/numba-0.61.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44240e694d4aa321430c97b21453e46014fe6c7b8b7d932afa7f6a88cc5d7e5e", size = 3890102 }, + { url = "https://files.pythonhosted.org/packages/ab/bc/dc2d03555289ae5263f65c01d45eb186ce347585c191daf0e60021d5ed39/numba-0.61.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:764f0e47004f126f58c3b28e0a02374c420a9d15157b90806d68590f5c20cc89", size = 3580239 }, + { url = "https://files.pythonhosted.org/packages/61/08/71247ce560d2c222d9ca705c7d3547fc4069b96fc85d71aabeb890befe9f/numba-0.61.0-cp313-cp313-win_amd64.whl", hash = "sha256:074cd38c5b1f9c65a4319d1f3928165f48975ef0537ad43385b2bd908e6e2e35", size = 2828035 }, +] + [[package]] name = "numexpr" version = "2.10.1" From c4a29b26dc624b288ff6852d5d7843113699b995 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:46:55 -0800 Subject: [PATCH 11/54] misc: let uv sync keep editable self install --- pyproject.toml | 4 ++++ uv.lock | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 44a93941..24a34871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + [project] name = "geodata-re" version = "0.1.0" diff --git a/uv.lock b/uv.lock index dcc5387e..0cf79794 100644 --- a/uv.lock +++ b/uv.lock @@ -792,7 +792,7 @@ wheels = [ [[package]] name = "geodata-re" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "boto3" }, { name = "bottleneck" }, From 122eda53f00d54e26774d0518b5d0abbe16f3a84 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:03:23 -0800 Subject: [PATCH 12/54] feat: incorporate hybrid levels into 3D dataset HRRR 3D Wind dataset now contains 10/80m AGL and 4 hybrid levels. --- src/geodata/datasets/hrrr/wind_3d.py | 40 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/geodata/datasets/hrrr/wind_3d.py b/src/geodata/datasets/hrrr/wind_3d.py index 5cab0261..25220bde 100644 --- a/src/geodata/datasets/hrrr/wind_3d.py +++ b/src/geodata/datasets/hrrr/wind_3d.py @@ -31,9 +31,20 @@ class HRRR3DWindHourlyDataset(HRRRBaseDataset): """HRRR3DWindHourlyDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. + + Variables: + - u_fixed: zonal wind speed at 10m and 80m + - v_fixed: meridional wind speed at 10m and 80m + - wind_speed_fixed: wind speed at 10m and 80m + - u_hybrid: zonal wind speed at hybrid levels (variable across locations) + - v_hybrid: meridional wind speed at hybrid levels (variable across locations) + + Important Coordinates: + - heightAboveGround: height above ground level of the fixed levels + - hybrid: hybrid level of the hybrid levels (fixed 1,2,3,4) """ - weather_config = "wind" + weather_config = "wind_3d" product = "nat" # Use "nat" product for 3D data def _download_file(self, file: AtomicDataset): @@ -58,9 +69,14 @@ def _download_file(self, file: AtomicDataset): with redirect_stdout_to_logger(logger, logging.INFO): logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") fh.download(":[UV]GRD:[1,8]0 m") + fh.download(":[UV]GRD:[1234] hybrid level") + fh.download(":HGT:[1234] hybrid level") uv_10 = [] uv_80 = [] + uv_hybrid = [] + hgt_hybrid = [] + for hour in date_range: h = Herbie( hour, @@ -72,17 +88,33 @@ def _download_file(self, file: AtomicDataset): try: uv_10.append( - h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"}) + h.xarray(":[UV]GRD:10 m", remove_grib=False).rename( + {"u10": "u", "v10": "v"} + ) + ) + uv_80.append(h.xarray(":[UV]GRD:80 m", remove_grib=False)) + uv_hybrid.append( + h.xarray(":[UV]GRD:[1234] hybrid level", remove_grib=False) + ) + hgt_hybrid.append( + h.xarray(":HGT:[1234] hybrid level", remove_grib=False) ) - uv_80.append(h.xarray("[UV]GRD:80 m")) except ValueError: logger.warning(f"No data found for {hour}, skipping.") uv_10 = xr.concat(uv_10, dim="time") uv_80 = xr.concat(uv_80, dim="time") + uv_hybrid = xr.concat(uv_hybrid, dim="time") + hgt_hybrid = xr.concat(hgt_hybrid, dim="time") ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") - ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + ds = ds.rename({"u": "u_fixed", "v": "v_fixed"}) + ds["wind_speed_fixed"] = (ds["u_fixed"] ** 2 + ds["u_fixed"] ** 2) ** 0.5 + + uv_hybrid = uv_hybrid.rename({"u": "u_hybrid", "v": "v_hybrid"}) + ds = xr.merge([ds, uv_hybrid, hgt_hybrid]) + + del uv_10, uv_80, uv_hybrid, hgt_hybrid try: del ds.attrs["search"] From 8e51eb9956408b646a216e76164e5b0a34c2d4aa Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 19 Feb 2025 01:41:37 +0000 Subject: [PATCH 13/54] misc: add devcontainer config --- .devcontainer/Dockerfile | 9 +++++++++ .devcontainer/devcontainer.json | 30 ++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..5266e394 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/devcontainers/base:bookworm + +USER root +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends libgdal-dev \ + && rm -rf /var/lib/apt/lists/* + +USER vscode +RUN curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..76ba6bc0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/debian +{ + "name": "Debian", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "build": { + // Path is relative to the devcontainer.json file. + "dockerfile": "Dockerfile" + }, + "postCreateCommand": "uv sync", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "charliermarsh.ruff" + ] + } + }, + "mounts": [ + "source=${localEnv:HOME}${localEnv:USERPROFILE}/.local/geodata,target=/home/vscode/.local/geodata,type=bind" + ] + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Configure tool-specific properties. + // "customizations": {}, + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} From b465928188ac721740905a0f847c67d34e6cafda Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 19 Feb 2025 19:35:03 +0000 Subject: [PATCH 14/54] wip: make cutout interoperable with new dataset --- src/geodata/convert.py | 229 +------------------- src/geodata/cutout.py | 298 ++++++++++---------------- src/geodata/datasets/_base.py | 42 ++-- src/geodata/datasets/era5/_base.py | 1 + src/geodata/datasets/merra2/_base.py | 1 + src/geodata/preparation.py | 309 +++++++++++++-------------- src/geodata/types.py | 2 +- 7 files changed, 285 insertions(+), 597 deletions(-) diff --git a/src/geodata/convert.py b/src/geodata/convert.py index f265b93f..6a5e0481 100644 --- a/src/geodata/convert.py +++ b/src/geodata/convert.py @@ -24,8 +24,6 @@ import numpy as np import xarray as xr -from six import string_types -from tqdm.auto import tqdm from . import wind as windm from .pv.irradiation import TiltedIrradiation @@ -36,117 +34,17 @@ logger = logging.getLogger(__name__) -def convert_cutout(cutout, convert_func, show_progress=False, **convert_kwds): - """ - Convert and aggregate a weather-based renewable generation time-series. - - NOTE: Not meant to be used by the user him or herself. Rather it is a - gateway function that is called by all the individual time-series - generation functions like pv and wind. Thus, all its parameters are also - available from these. - - Parameters (passed through as **params) - --------------------------------------- - show_progress : boolean|string - Whether to show a progress bar if boolean and its label if given as a - string (defaults to True). - - Returns - ------- - resource : xr.DataArray - Time-series of renewable generation aggregated to buses, if - `matrix` or equivalents are provided else the total sum of - generated energy. - - Internal Parameters (provided by f.ex. wind and pv) - --------------------------------------------------- - convert_func : Function - Callback like convert_wind, convert_pv - """ - if not cutout.prepared: - raise RuntimeError("The cutout has to be prepared first.") - - results = [] - - yearmonths = cutout.coords["year-month"].to_index() - - if isinstance(show_progress, string_types): - prefix = show_progress - else: - func_name = ( - convert_func.__name__[len("convert_") :] - if convert_func.__name__.startswith("convert_") - else convert_func.__name__ - ) - prefix = f"Convert `{func_name}`: " - - pbar = tqdm if show_progress else lambda x, desc: x - for ym in pbar(yearmonths, desc=prefix): - with xr.open_dataset(cutout.datasetfn(ym)) as ds: - if "view" in cutout.meta.attrs: - if isinstance(cutout.meta.attrs["view"], str): - cutout.meta.attrs["view"] = {} - cutout.meta.attrs.setdefault("view", {})["x"] = slice( - min(cutout.meta.coords["x"]).values.tolist(), - max(cutout.meta.coords["x"]).values.tolist(), - ) - cutout.meta.attrs.setdefault("view", {})["y"] = slice( - min(cutout.meta.coords["y"]).values.tolist(), - max(cutout.meta.coords["y"]).values.tolist(), - ) - ds = ds.sel(**cutout.meta.attrs["view"]) - - da = convert_func(ds, **convert_kwds) - results.append(da.load()) - - results = xr.concat(results, dim="time") - - return results - - -## temperature - - -def convert_temperature(ds): - """Return outside temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - """ - - # Temperature is in Kelvin - return ds["temperature"] - 273.15 - - -def temperature(cutout, **params): - return cutout.convert_cutout(convert_func=convert_temperature, **params) - - -## soil temperature - - -def convert_soil_temperature(ds): - """Return soil temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - """ - - # Temperature is in Kelvin - - # There are nans where there is sea; by setting them - # to zero we guarantee they do not contribute when multiplied - # by matrix in geodata/aggregate.py - return (ds["soil temperature"] - 273.15).fillna(0.0) - - -def soil_temperature(cutout, **params): - return cutout.convert_cutout(convert_func=convert_soil_temperature, **params) - - -## heat demand - - -def convert_heat_demand(ds, threshold, a, constant, hour_shift): +# Heat Demand +def convert_heat_demand( + ds: xr.Dataset, + threshold: float, + a: float, + constant: float, + hour_shift: float, +): # Temperature is in Kelvin; take daily average T = ds["temperature"] - T.coords["time"].values += np.timedelta64(dt.timedelta(hours=hour_shift)) + T.coords["time"] += np.timedelta64(dt.timedelta(hours=hour_shift)) T = ds["temperature"].resample(time="1D").mean(dim="time") threshold += 273.15 @@ -157,62 +55,6 @@ def convert_heat_demand(ds, threshold, a, constant, hour_shift): return constant + heat_demand_value -def heat_demand(cutout, threshold=15.0, a=1.0, constant=0.0, hour_shift=0.0, **params): - """ - Convert outside temperature into daily heat demand using the - degree-day approximation. - - Since "daily average temperature" means different things in - different time zones and since xarray coordinates do not handle - time zones gracefully like pd.DateTimeIndex, you can provide an - hour_shift to redefine when the day starts. - - E.g. for Moscow in winter, hour_shift = 4, for New York in winter, - hour_shift = -5 - - This time shift applies across the entire spatial scope of ds for - all times. More fine-grained control will be built in a some - point, i.e. space- and time-dependent time zones. - - WARNING: Because the original data is provided every month, at the - month boundaries there is untidiness if you use a time shift. The - resulting xarray will have duplicates in the index for the parts - of the day in each month at the boundary. You will have to - re-average these based on the number of hours in each month for - the duplicated day. - - Parameters - ---------- - threshold : float - Outside temperature in degrees Celsius above which there is no - heat demand. - a : float - Linear factor relating heat demand to outside temperature. - constant : float - Constant part of heat demand that does not depend on outside - temperature (e.g. due to water heating). - hour_shift : float - Time shift relative to UTC for taking daily average - - Note - ---- - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - """ - - return cutout.convert_cutout( - convert_func=convert_heat_demand, - threshold=threshold, - a=a, - constant=constant, - hour_shift=hour_shift, - **params, - ) - - -## solar thermal collectors - - def convert_solar_thermal( ds, orientation, trigon_model, clearsky_model, c0, c1, t_store ): @@ -249,9 +91,6 @@ def convert_pv(ds, panel, orientation, trigon_model="simple", clearsky_model="si return solar_panel -## wind - - def convert_wind(ds, turbine, **params): """ Convert wind speeds for turbine to wind energy generation. @@ -271,7 +110,6 @@ def convert_wind(ds, turbine, **params): """ V, POW, hub_height, P = itemgetter("V", "POW", "hub_height", "P")(turbine) - wnd_hub = windm.extrapolate_wind_speed(ds, to_height=hub_height, **params) return xr.DataArray(np.interp(wnd_hub, V, POW / P), coords=wnd_hub.coords) @@ -355,52 +193,3 @@ def convert_pm25(ds): ) return 1e9 * ds["pm25"] # kg / m3 to ug / m3 - - -# Manipulate arbitrary variables - - -def _get_var(ds, var): - """ - (Internal) Extract a specific variable from cutout - See: get_var - """ - return xr.DataArray(ds[var], coords=ds.coords) - - -def get_var(cutout, var, **params): - """ - Extract a specific variable from cutout - - Parameters - ---------- - var : str - Name of variable to extract from dataset - - Returns: dataarray - """ - logger.info("Getting variable: %s", str(var)) - return cutout._convert_cutout(convert_func=_get_var, var=var, **params) - - -def _compute_var(ds, fn): - """ - (Internal) Compute a specific function from cutout - See: compute_var - """ - return xr.DataArray(fn(ds), coords=ds.coords) - - -def compute_var(cutout, fn, **params): - """ - Compute a specific function from cutout - - Parameters - ---------- - var : str - Name of variable to extract from dataset - - Returns: dataarray - """ - logger.info("Computing variable: %s", str(fn)) - return cutout.convert_cutout(convert_func=_compute_var, fn=fn, **params) diff --git a/src/geodata/cutout.py b/src/geodata/cutout.py index 058f2c33..1f020e6e 100644 --- a/src/geodata/cutout.py +++ b/src/geodata/cutout.py @@ -1,6 +1,6 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) # Copyright 2020 Michael Davidson (UCSD), William Honaker, Jiahe Feng (UCSD), Yuanbo Shi -# Copyright 2023-2024 Xiqiang Liu +# Copyright 2023-2025 Xiqiang Liu # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -19,15 +19,11 @@ Cutout class to handle a subset of a Dataset. """ -import datetime as dt import logging -import os -import sys -from collections.abc import Iterable from functools import partial from operator import itemgetter from pathlib import Path -from typing import Literal, Optional, Union +from typing import Literal, Optional, Sequence, Union import numpy as np import pyproj @@ -36,9 +32,9 @@ from shapely.geometry import box from tqdm.auto import tqdm - from . import config from .convert import ( + convert_heat_demand, convert_pm25, convert_pv, convert_solar_thermal, @@ -47,12 +43,7 @@ convert_windwpd, get_orientation, ) - -from .resource import ( - get_solarpanelconfig, - get_windturbineconfig, - windturbine_smooth, -) +from .datasets._base import BaseDataset from .mask import Mask from .preparation import ( cutout_get_meta, @@ -60,6 +51,11 @@ cutout_prepare, cutout_produce_specific_dataseries, ) +from .resource import ( + get_solarpanelconfig, + get_windturbineconfig, + windturbine_smooth, +) logger = logging.getLogger(__name__) @@ -68,13 +64,15 @@ class Cutout: """Cutout class to handle a subset of a Dataset. Args: - module (Literal["era5", "merra2"]): name of the dataset module to use. - weather_data_config (str): name of the weather data config to use. - the name will be automatically generated. + name (str): Name of the cutout. This name will be used to uniquely + identify the cutout. If a cutout with the same name already exists, + the existing cutout will be retrieved. + dataset_cls (type[BaseDataset]): Dataset class to use for the cutout. years (slice): years of the cutout. - name (Optional[str]): name of the cutout. Optional. If not specified, - cutout_dir (str): path to the cutout directory. Defaults to config.cutout_dir. - bounds (Optional[Iterable]): bounds of the cutout. Optional. If not specified, + cutout_dir (str): path to the cutout directory. This is optional. If not + specified, the cutout will be stored in the default cutout directory + under `GEODATA_ROOT`. + bounds (Optional[Sequence]): bounds of the cutout. Optional. If not specified, the bounds will be automatically generated. months (Optional[slice]): months of the cutout. Optional. If not specified, the months will be automatically generated. @@ -86,29 +84,28 @@ class Cutout: def __init__( self, - module: Literal["era5", "merra2"], - weather_data_config: str, + name: str, + dataset_cls: type[BaseDataset], years: slice, - name: Optional[str] = None, cutout_dir: Union[str, Path] = config.cutout_dir, - bounds: Optional[Iterable] = None, + bounds: Optional[Sequence] = None, months: Optional[slice] = None, xs: Optional[slice] = None, ys: Optional[slice] = None, ): self.name = name - self.cutout_dir = os.path.join(cutout_dir, name) + self.cutout_dir = Path(cutout_dir, name) + + self.dataset_cls = dataset_cls + self.prepared = False self.empty = False - self.meta_append = 0 - self.config = weather_data_config self.meta = None self.merged_mask = None self.shape_mask = None self.area = None params_dict = { - "module": module, "years": years, "months": months, "xs": xs, @@ -117,6 +114,7 @@ def __init__( if bounds is not None and (xs is not None or ys is not None): raise TypeError("Cannot specify both bounds and xs/ys arguments.") + if bounds is not None: # if passed bounds array instead of xs, ys slices x1, y1, x2, y2 = bounds @@ -126,39 +124,15 @@ def __init__( logger.info("No months specified, defaulting to 1-12") params_dict.update(months=slice(1, 12)) - if os.path.isdir(self.cutout_dir): + if self.cutout_dir.is_dir(): + # Check if cutout directory exists # If cutout dir exists, check completness of files - if os.path.isfile(self.datasetfn()): # open existing meta file - self.meta = xr.open_dataset(self.datasetfn()).stack( - **{"year-month": ("year", "month")} + if self.meta_path.is_file(): # open existing meta file + self.meta = xr.open_dataset(self.get_filename()).stack( + dim={"year-month": ("year", "month")} ) - if ( - self.meta is not None - and "years" in params_dict - and "months" in params_dict - and all( - os.path.isfile(self.datasetfn([y, m])) - for y in range( - params_dict["years"].start, params_dict["years"].stop + 1 - ) - for m in range( - params_dict["months"].start, params_dict["months"].stop + 1 - ) - ) - ): - # All files are accounted for. Checking basic data and coverage - if "module" not in self.meta.attrs: - raise TypeError("No module given in meta file of cutout.") - - # load dataset module based on file metadata - from geodata import Dataset - - self.dataset_module: Dataset = sys.modules[ - "geodata.datasets." + self.meta.attrs["module"] - ] - params_dict["module"] = self.meta.attrs["module"] - + if all(self.catalog): logger.info("All cutout (%s, %s) files available.", name, cutout_dir) # At least one of xs, ys is in params_dict @@ -181,118 +155,94 @@ def __init__( self.prepared = False logger.info("Cutout (%s, %s) not complete.", name, cutout_dir) + # In case if not self.prepared: - # Still need to prepare cutout - if "module" not in params_dict: - raise TypeError("Module is required to create cutout.") - # load module from geodata library - self.dataset_module = sys.modules[ - "geodata.datasets." + params_dict["module"] - ] - logger.info("Cutout (%s, %s) not found or incomplete.", name, cutout_dir) if {"xs", "ys", "years"}.difference(params_dict): raise TypeError( - "Arguments `xs`, `ys`, and `years` need to be specified for a cutout." + "Arguments `xs`, `ys`, and `years` need to be specified to create " + "a cutout." ) if self.meta is not None: # if meta.nc exists, close and delete it self.meta.close() - os.remove(self.datasetfn()) + self.meta_path.unlink() - ## Main preparation call for metadata - # preparation.cutout_get_meta - # cutout.meta_data_config - # dataset_module.meta_data_config (e.g. prepare_meta_era5) self.meta = self.get_meta(**params_dict) - - # Ensure cutout directory exists - if not os.path.isdir(self.cutout_dir): - os.mkdir(self.cutout_dir) + self.cutout_dir.mkdir(parents=True, exist_ok=True) # Write meta file - self.meta_clean.unstack("year-month").to_netcdf(self.datasetfn()) + self.meta_clean.unstack("year-month").to_netcdf(self.meta_path) - def datasetfn(self, *args): + def _get_filename(self, year: int | None = None, month: int | None = None): """Return path to dataset xarray files related to this Cutout. + If both year and month are None, return path to meta.nc file. Args: - *args: optional arguments to append to the filename. If not specified, - the meta file will be returned. If specified, the dataset file will be returned, depending - on the number of arguments. One argument will return the dataset file for the given - year-month string, two arguments will return the dataset file for the given year and month. + year (int): The year to get the filename for. Defaults to None. + month (int): The month to get the filename for. Defaults to None. Returns: - str: path to dataset xarray files related to this Cutout. + Path: Path to the dataset xarray file. """ - dataset = None - if len(args) == 2: - dataset = args - elif len(args) == 1: - dataset = args[0] - else: - dataset = None - return os.path.join( - # pylint: disable=consider-using-f-string - self.cutout_dir, - ("meta.nc" if dataset is None else "{}{:0>2}.nc".format(*dataset)), - # pylint: enable=consider-using-f-string - ) + if year is None and month is None: + return self.cutout_dir / "meta.nc" + + if year is not None: + if month is not None: + return self.cutout_dir / f"{year}{month:0>2}.nc" + + return self.cutout_dir / f"{year}.nc" + + @property + def catalog(self): + """A generator that yields all dataset files.""" + + for year in self.coords["year"]: + for month in self.coords["month"]: + yield self._get_filename(year, month) + + @property + def meta_path(self): + """Path to the metadata file.""" + return self._get_filename() @property def meta_data_config(self): """Metadata configuration for the Cutout""" - return dict( - tasks_func=self.dataset_module.weather_data_config[self.config][ - "tasks_func" - ], - prepare_func=self.dataset_module.weather_data_config[self.config][ - "meta_prepare_func" - ], - template=self.dataset_module.weather_data_config[self.config]["template"], - file_granularity=self.dataset_module.weather_data_config[self.config][ - "file_granularity" - ], - ) + return { + "tasks_func": self.dataset_cls.tasks_func, + "prepare_func": self.dataset_cls.meta_prepare_func, + "file_granularity": self.dataset_cls.frequency, + } @property - def weather_data_config(self): + def weather_config(self): """The weather data configuration for the Cutout.""" - return self.dataset_module.weather_data_config - @property - def variables(self): - """The variables contained in the Cutout.""" - return self.dataset_module.weather_data_config[self.config]["variables"] + return self.dataset_cls.weather_config @property def info(self): """Summary information about the Cutout.""" - return dict( - name=self.name, - config=self.config, - prepared=self.prepared, - projection=self.dataset_module.projection, - shape=[len(self.coords["y"]), len(self.coords["x"])], - extent=( - list(self.coords["x"].values[[0, -1]]) - + list(self.coords["y"].values[[-1, 0]]) - ), - dimensions=self.meta.dims, - coordinates=self.meta.coords, - variables=self.dataset_module.weather_data_config[self.config]["variables"], - dataset_module=self.dataset_module, - cutout_dir=self.cutout_dir, - ) + return { + "name": self.name, + "prepared": self.prepared, + "shape": self.shape, + "extent": self.extent, + "years": self.years, + "months": self.months, + "meta": self.meta, + } @property def projection(self): """The projection of the Cutout.""" - return self.dataset_module.projection + return self.dataset_cls.projection @property def coords(self): @@ -305,7 +255,7 @@ def meta_clean(self): meta = self.meta if meta.attrs.get("view", {}): view = {} - for name, value in meta.attrs.get("view", {}).items(): + for name, value in meta.attrs["view"].items(): view.update({name: [value.start, value.stop]}) meta.attrs["view"] = str(view) return meta @@ -345,7 +295,6 @@ def grid_cells(self): def __repr__(self): yearmonths = self.coords["year-month"].to_index() - # pylint: disable=consider-using-f-string return "".format( self.name, self.coords["x"].values[0], @@ -358,7 +307,6 @@ def __repr__(self): yearmonths[-1][1], "" if self.prepared else "UN", ) - # pylint: enable=consider-using-f-string def add_mask(self, name: str, merged_mask: bool = True, shape_mask: bool = True): """Add mask attribute to the cutout, from a previously saved mask objects. @@ -500,7 +448,7 @@ def mask( return res # Preparation functions - get_meta = cutout_get_meta # preparation.cutout_get_meta + get_meta = cutout_get_meta get_meta_view = cutout_get_meta_view # preparation.cutout_get_meta_view prepare = cutout_prepare # preparation.cutout_prepare produce_specific_dataseries = cutout_produce_specific_dataseries @@ -539,12 +487,12 @@ def _convert_cutout( if convert_func.__name__.startswith("convert_") else convert_func.__name__ ) - prefix = f"Convert `{func_name}`: " + prefix = f"Convert {func_name}" for ym in tqdm( yearmonths, desc=prefix, disable=not show_progress, dynamic_ncols=True ): - with xr.open_dataset(self.datasetfn(ym)) as ds: + with xr.open_dataset(self._get_filename(ym)) as ds: if "view" in self.meta.attrs: if isinstance(self.meta.attrs["view"], str): self.meta.attrs["view"] = {} @@ -608,25 +556,6 @@ def heat_demand( documented in the `convert_cutout` function. """ - def convert_heat_demand( - ds: xr.Dataset, - threshold: float, - a: float, - constant: float, - hour_shift: float, - ): - # Temperature is in Kelvin; take daily average - T = ds["temperature"] - T.coords["time"].values += np.timedelta64(dt.timedelta(hours=hour_shift)) - - T = ds["temperature"].resample(time="1D").mean(dim="time") - threshold += 273.15 - heat_demand_value = a * (threshold - T) - - heat_demand_value.values[heat_demand_value.values < 0.0] = 0.0 - - return constant + heat_demand_value - return self._convert_cutout( convert_func=convert_heat_demand, threshold=threshold, @@ -715,33 +644,22 @@ def solar_thermal( **params, ) - # NOTE: The following wind-related functions will be deprecated in the future - # in favor of the wind modeling module. def wind( - self, turbine: Union[str, dict], smooth: Union[bool, dict] = False, **params + self, + turbine: Union[str, dict], + method: Literal["simple", "interpolation", "extrapolation"], + smooth: Union[bool, dict] = False, + **params, ): - """ - Generate wind generation time-series - - - loads turbine dict based on passed parameters (resource.get_windturbineconfig) - - optionally, smooths turbine power curve (resource.windturbine_smooth) - - calls convert_wind (convert.convert_cutout) + """Convert wind speed time-series into wind generation time-series. Args: - turbine (Union[str, dict]): Name of a turbine known by the reatlas client or a - turbineconfig dictionary with the keys 'hub_height' for the - hub height and 'V', 'POW' defining the power curve. - smooth (Union[bool, dict]): If True smooth power curve with a gaussian kernel as - determined for the Danish wind fleet to Delta_v = 1.27 and - sigma = 2.s29. A dict allows to tune these values. - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Andresen G B, Søndergaard A A and Greiner M 2015 Energy 93, Part 1 - 1074 - 1088. doi:10.1016/j.energy.2015.09.071 + turbine (Union[str, dict]): Name of a turbine or a dictionary with the parameters + for the wind turbine in [2]. + smooth (Union[bool, dict]): If True, the wind speed time-series will be smoothed + before conversion. If False, no smoothing will be applied. If a dictionary is + passed, the smoothing parameters will be used. + **params: Keyword arguments passed to `convert_cutout` function """ if isinstance(turbine, str): @@ -750,9 +668,14 @@ def wind( if smooth: turbine = windturbine_smooth(turbine, params=smooth) - return self._convert_cutout( - convert_func=convert_wind, turbine=turbine, **params - ) + match method: + case "simple": + return self._convert_cutout( + convert_func=convert_wind, turbine=turbine, **params + ) + + case _: + raise ValueError(f"Method {method} not supported.") def windspd(self, **params): """ @@ -892,13 +815,8 @@ def pm25(self, **params): Generate PM2.5 time series [ug / m3] (see convert_pm25 for details) - Parameters - ---------- - **params : None needed currently. - - Returns - ------- - pm25 : xr.DataArray + Returns: + xr.DataArray: PM2.5 time series """ @@ -946,7 +864,7 @@ def _find_intercept(list1, list2, start, threshold=0): if min_res == init: return 0 else: - return i # type: ignore + return i def coarsen(ori: xr.Dataset, tar: xr.Dataset, func: Literal["sum", "mean"] = "mean"): @@ -998,7 +916,7 @@ def coarsen(ori: xr.Dataset, tar: xr.Dataset, func: Literal["sum", "mean"] = "me else: raise ValueError("func can only be 'mean' or 'sum'") - return coarsen.reindex_like(tar, method="nearest") + return _coarsen.reindex_like(tar, method="nearest") def calc_grid_area(lis_lats_lons): diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 225b7789..5edde962 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -350,6 +350,14 @@ def projection(self): represents the projection of the dataset. """ + @property + @abc.abstractmethod + def lat_direction(self) -> bool: + """Latitude direction stored in the dataset. This should be a boolean flag. + If True, the latitude increases from south to north. If False, the latitude + increases from north to south. + """ + @property def extra_repr(self): return "" @@ -376,6 +384,16 @@ def submodule(self): """ return getattr(self, "weather_config", self.__class__.__name__) + @staticmethod + @abc.abstractmethod + def tasks_func(): + """A method that returns a list of tasks that can be run on the dataset.""" + + @staticmethod + @abc.abstractmethod + def meta_prepare_func(): + """A method that returns a list of metadata preparation tasks.""" + @property def catalog(self) -> list["AtomicDataset"]: """A generator that yields all the files that need to be downloaded. @@ -429,30 +447,6 @@ def _daily_catalog(self) -> list["AtomicDataset"]: return catalog - # def _hourly_catalog(self): - # catalog = [] - - # for year, month in itertools.product( - # range(self.years.start, self.years.stop + 1), - # range(self.months.start, self.months.stop + 1), - # ): - # for day in range(1, pd.Timestamp(f"{year}-{month}-1").days_in_month + 1): - # for hour in range(24): - # save_path = ( - # self.storage_root - # / f"{year}_{month:02d}_{day:02d}_{hour:02d}.nc" - # ) - # catalog.append( - # { - # "year": year, - # "month": month, - # "day": day, - # "hour": hour, - # "save_path": save_path, - # } - # ) - # return catalog - def _rename_and_clean_coords(self, ds: xr.Dataset, add_lon_lat: bool = False): """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index 631fee09..670af3d4 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -28,6 +28,7 @@ class ERA5BaseDataset(BaseDataset): module = "era5" projection = "latlong" + lat_direction = False def _extra_setup(self, **kwargs): self.logger = logging.getLogger(__name__.replace("._base", ".client")) diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 6e33981f..e3492d15 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -32,6 +32,7 @@ class MERRA2BaseDataset(BaseDataset): module = "merra2" projection = "latlong" + lat_direction = True frequency = "daily" url_template = "" diff --git a/src/geodata/preparation.py b/src/geodata/preparation.py index 3a902be0..5226668b 100644 --- a/src/geodata/preparation.py +++ b/src/geodata/preparation.py @@ -1,4 +1,5 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,7 +15,7 @@ # along with this program. If not, see . -"""Various Preparation Functions for Geodata's modules""" +"""Various Preparation Functions for Geodata's modules.""" import calendar import logging @@ -22,27 +23,32 @@ import shutil import subprocess import tempfile -from glob import glob from multiprocessing import Pool +from typing import TYPE_CHECKING import dask import numpy as np import pandas as pd import xarray as xr +if TYPE_CHECKING: + from geodata.cutout import Cutout + logger = logging.getLogger(__name__) -def cutout_do_task(task, write_to_file=True): +def cutout_do_task(task: dict, write_to_file: bool = True): task = task.copy() prepare_func = task.pop("prepare_func") + if write_to_file: - datasetfns = task.pop("datasetfns") + datasetfns = task.pop("filenames") # Force dask to use just one thread (to save memory) with dask.config.set(scheduler="single-threaded"): try: data = prepare_func(**task) + if data is None: data = [] @@ -50,8 +56,10 @@ def cutout_do_task(task, write_to_file=True): for yearmonth, ds in data: if ds is None: continue - fn = datasetfns[yearmonth] # type: ignore + + fn = datasetfns[yearmonth] logger.debug("Writing to %s", os.path.basename(fn)) + ds.to_netcdf(fn) logger.debug( "Write variable(s) %s to %s generated by %s", @@ -59,8 +67,10 @@ def cutout_do_task(task, write_to_file=True): os.path.basename(fn), prepare_func.__name__, ) - else: - return data + return + + return data + except Exception as e: logger.exception( "Exception occured in the task with prepare_func `%s`: %s", @@ -70,19 +80,32 @@ def cutout_do_task(task, write_to_file=True): raise e -def cutout_prepare(cutout, overwrite=False, nprocesses=None, gebco_height=False): - """ - Main preparation function +def cutout_prepare( + cutout: Cutout, + overwrite: bool = False, + nprocesses: int | None = None, + gebco_height: bool = False, +): + """Main preparation function for a cutout. + + Args: + cutout: The cutout object to prepare. + overwrite: If True, the cutout will be prepared even if it is already prepared. + nprocesses: Number of processes to use. If None, all processors will be used. + gebco_height: If True, the function will interpolate the GEBCO bathymetry to the dataset grid. """ + if cutout.prepared and not overwrite: logger.info( - "The cutout is already prepared. If you want to recalculate it, supply an `overwrite=True` argument." + "The cutout is already prepared. If you want to recalculate it" + ", supply an `overwrite=True` argument." ) return True - if cutout.empty is True: + if cutout.empty: logger.warning( - "Cutout dimensions are empty. One or more of xs, ys, or time are not available in Dataset." + "Cutout dimensions are empty. One or more of xs, ys, or time are not " + "available in Dataset." ) return False @@ -93,59 +116,34 @@ def cutout_prepare(cutout, overwrite=False, nprocesses=None, gebco_height=False) xs = cutout.meta.indexes["x"] ys = cutout.meta.indexes["y"] - # 1. (here) uncomment and change yearmonths to only new ones. delete yearmonths line above - # 2. (elsewhere) uncomment meta_append / append lines - - # if cutout.meta_append == 1: - # # appended meta. prepare only new year-months - # yearmonths = cutout.coords['year-month'].to_index() - # else: - # yearmonths = cutout.coords['year-month'].to_index() - if gebco_height: logger.info("Interpolating gebco to the dataset grid") cutout.meta["height"] = _prepare_gebco_height(xs, ys) # Check if cutout_dir exists - if os.path.isdir(cutout_dir): + if cutout_dir.is_dir(): # Delete all files except meta.nc - logger.debug("Deleting cutout files in '%s'", cutout_dir) - for delete_file in glob(os.path.join(cutout_dir, "*.*")): - if not delete_file.endswith("meta.nc"): - logger.debug(delete_file) - os.remove(delete_file) - # shutil.rmtree(cutout_dir) - else: - os.mkdir(cutout_dir) - - # # Moved to cutout __init__: Write meta file - # (cutout.meta_clean - # .unstack('year-month') - # .to_netcdf(cutout.datasetfn())) + for f in cutout_dir.rglob("*.nc"): + if not f.name.endswith("meta.nc"): + logger.debug("Deleting %s", f.name) + f.unlink() + cutout_dir.mkdir(parents=True, exist_ok=True) # Compute data and fill files - tasks = [] - - # for series in itervalues(cutout.weather_data_config[cutout.config]): - # dict of tasks w/structure (tasks_func, prepare_func) - # .. could be one task and prepare (eg prepare_month_era5) - series = cutout.weather_data_config[cutout.config] - series["meta_attrs"] = cutout.meta.attrs - tasks_func = series["tasks_func"] + tasks = cutout.dataset_cls.tasks_func(xs=xs, ys=ys, yearmonths=yearmonths) - # form call to task_func (eg tasks_monthly_merra2) - # .. **series contains prepare_func - # returns: dict(prepare_func=prepare_func, xs=xs, ys=ys, year=year, month=month) - # .. or: dict(prepare_func=prepare_func, xs=xs, ys=ys, fn=next(glob...), engine=engine, yearmonth=ym) - tasks += tasks_func(xs=xs, ys=ys, yearmonths=yearmonths, **series) for i, t in enumerate(tasks): def datasetfn_with_id(ym): # returns a filename with incrementing id at end eg `201101-01.nc` - base, ext = os.path.splitext(cutout.datasetfn(ym)) + filename = cutout._get_filename(ym) + + base = filename.stem + ext = filename.suffix + return f"{base}-{i}{ext}" - t["datasetfns"] = {ym: datasetfn_with_id(ym) for ym in yearmonths.tolist()} + t["filenames"] = {ym: datasetfn_with_id(ym) for ym in yearmonths.tolist()} # TODO: Using multiple processes will cause issues with geodata currently # because geodata write the metadata dataframe to the cutout directory @@ -163,39 +161,37 @@ def datasetfn_with_id(ym): # f"{nprocesses} processes" if nprocesses is not None else "all processors", # ) - pool = Pool(processes=1) - try: - pool.map(cutout_do_task, tasks) - except Exception as e: - pool.terminate() - logger.info( - "Preparation of cutout '%s' has been interrupted by an exception. " - "Purging the incomplete cutout_dir.", - cutout.name, - ) - shutil.rmtree(cutout_dir) - raise e - pool.close() + with Pool(processes=nprocesses) as pool: + try: + pool.map(cutout_do_task, tasks) + except Exception as e: + pool.terminate() + logger.info( + "Preparation of cutout '%s' has been interrupted by an exception. " + "Purging the incomplete cutout_dir.", + cutout.name, + ) + shutil.rmtree(cutout_dir) + raise e logger.info("Merging variables into monthly compound files") - for fn in map(cutout.datasetfn, yearmonths.tolist()): + for fn in [cutout._get_filename(ym) for ym in yearmonths.tolist()]: # Find all files with yearmonth prefix eg `201101-XX.nc` - base, ext = os.path.splitext(fn) - fns = glob(base + "-*" + ext) + fns = list(cutout_dir.rglob(f"{fn.stem}-*.{fn.suffix}")) + if len(fns) == 1 and not gebco_height: # Just a single file. Simply rename - os.rename(fns[0], fn) + fns[0].rename(fn) else: - # Multiple files for yearmonth - # open_mfdataset: auto-magically determines appropriate concat and merge of datasets - with xr.open_mfdataset(fns, combine="by_coords") as ds: + with xr.open_mfdataset(fns, combine="by_coords", chunks="auto") as ds: if gebco_height: ds["height"] = cutout.meta["height"] ds.to_netcdf(fn) - for tfn in fns: - os.unlink(tfn) + for f in fns: + f.unlink() + logger.debug("Completed files %s", os.path.basename(fn)) logger.info("Cutout '%s' has been successfully prepared", cutout.name) @@ -216,102 +212,26 @@ def cutout_produce_specific_dataseries(cutout, yearmonth, series_name): return data[0][1] # type: ignore -def cutout_get_meta(cutout, xs, ys, years, months=None, **dataset_params): - # called in cutout.py as `get_meta()` - # Loads various metadata (coordinates, dims...) from dataset via dataset_module.prepare_func (eg prepare_meta_merra2) - - if months is None: - months = slice(1, 12) - - ys = _prepare_lat_direction(cutout.dataset_module.lat_direction, ys) - - meta_kwds = cutout.meta_data_config.copy() - meta_kwds.update(dataset_params) - - # Assign task function here? - tasks_func = meta_kwds["tasks_func"] # noqa: F841 - # test before removing - - # Get metadata - prepare_func = meta_kwds.pop("prepare_func") - ds = prepare_func(xs=xs, ys=ys, year=years.stop, month=months.stop, **meta_kwds) - ds.attrs.update(dataset_params) - - # Check if cutout dimenions are empty - dims = [len(ds.indexes[i]) for i in ds.indexes] - if np.prod(dims) == 0: - logger.warning( - "Cutout dimensions are empty. One or more of xs, ys, or time are not available in Dataset." - ) - cutout.empty = True - - # with metadata, load various parameters - meta_file_granularity = meta_kwds["file_granularity"] - month_start = pd.Timestamp(f"{years.stop}-{months.stop}") - ds.coords["year"] = range(years.start, years.stop + 1) - ds.coords["month"] = range(months.start, months.stop + 1) - - if meta_file_granularity == "daily": - start, second, end = map(pd.Timestamp, ds.coords["time"].values[[0, 1, -1]]) - offset_start = start - month_start - offset_end = end - (month_start + pd.offsets.MonthBegin()) - step = (second - start).components.hours - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}") + offset_start, - end=(month_start + pd.offsets.MonthBegin() + offset_end), - freq="h" if step == 1 else f"{step}h", - ) - elif meta_file_granularity == "dailymeans": - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}-1"), - end=pd.Timestamp( - f"{years.stop}-{months.stop}-{calendar.monthrange(years.stop, months.stop)[1]}" - ), - freq="d", - ) - elif meta_file_granularity == "monthly": - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}"), - end=pd.Timestamp(f"{years.stop}-{months.stop}"), - freq="MS", - ) - - ds = ds.stack(**{"year-month": ("year", "month")}) - - # if cutout.meta_append == 1: - # # Append to existing meta - # (ds - # .unstack('year-month') - # .to_netcdf(cutout.datasetfn('meta','2')) ) - # - # with xr.open_mfdataset([cutout.datasetfn(),cutout.datasetfn('meta','2')], combine='by_coords') as ds_comb: - # ds = ds_comb - # ds = ds.stack(**{'year-month': ('year', 'month')}) - - return ds - - def cutout_get_meta_view( - cutout, xs=None, ys=None, years=slice(None), months=slice(None), **dataset_params + cutout: Cutout, + xs: slice | None = None, + ys: slice | None = None, + years=slice(None), + months=slice(None), + **dataset_params, ): - # called in cutout as `get_meta_view()` - # Create subset of metadata based on xs, ys, years, months - # Returns None if any of the dimensions of the subset are empty - meta = cutout.meta meta.attrs["view"] = {} if xs is not None: - meta.attrs.setdefault("view", {})["x"] = xs + meta.attrs["x"] = xs if ys is not None: - meta.attrs.setdefault("view", {})["y"] = _prepare_lat_direction( - cutout.dataset_module.lat_direction, ys - ) + meta.attrs["y"] = _prepare_lat_direction(cutout.dataset_cls.lat_direction, ys) meta = ( meta.unstack("year-month") .sel(year=years, month=months, **meta.attrs.get("view", {})) - .stack(**{"year-month": ("year", "month")}) + .stack(dim={"year-month": ("year", "month")}) ) meta = meta.sel( @@ -328,9 +248,73 @@ def cutout_get_meta_view( dim_len = [len(meta.indexes[i]) for i in meta.indexes] if all(d > 0 for d in dim_len): return meta - else: - logger.info(dim_len) - return None + + logger.warning( + "Certain dimensions are empty with the meta file of the cutout %s", dim_len + ) + + +def cutout_get_meta( + cutout: Cutout, + xs: slice, + ys: slice, + years: slice, + months: slice = slice(1, 12), + **dataset_params, +) -> xr.Dataset: + ys = _prepare_lat_direction(cutout.dataset_cls.lat_direction, ys) + + meta_kwds = cutout.meta_data_config.copy() + meta_kwds.update(dataset_params) + + # Get metadata + prepare_func: callable = meta_kwds.pop("prepare_func") + ds: xr.Dataset = prepare_func( + xs=xs, ys=ys, year=years.stop, month=months.stop, **meta_kwds + ) + ds.attrs.update(dataset_params) + + # Check if cutout dimenions are empty + if not np.prod([len(ds.indexes[i]) for i in ds.indexes]): + logger.warning( + "Cutout dimensions are empty. " + "One or more of xs, ys, or time are not available in Dataset." + ) + cutout.empty = True + + # with metadata, load various parameters + meta_file_granularity = meta_kwds["file_granularity"] + month_start = pd.Timestamp(f"{years.stop}-{months.stop}") + ds.coords["year"] = range(years.start, years.stop + 1) + ds.coords["month"] = range(months.start, months.stop + 1) + + match meta_file_granularity: + case "daily": + start, second, end = map(pd.Timestamp, ds.coords["time"].values[[0, 1, -1]]) + offset_start = start - month_start + offset_end = end - (month_start + pd.offsets.MonthBegin()) + step = (second - start).components.hours + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}") + offset_start, + end=(month_start + pd.offsets.MonthBegin() + offset_end), + freq="h" if step == 1 else f"{step}h", + ) + case "dailymeans": + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}-1"), + end=pd.Timestamp( + f"{years.stop}-{months.stop}-{calendar.monthrange(years.stop, months.stop)[1]}" + ), + freq="d", + ) + case "monthly": + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}"), + end=pd.Timestamp(f"{years.stop}-{months.stop}"), + freq="MS", + ) + + return ds.stack(**{"year-month": ("year", "month")}) def _prepare_gebco_height(xs, ys, gebco_fn=None): @@ -387,11 +371,12 @@ def _prepare_gebco_height(xs, ys, gebco_fn=None): return height -def _prepare_lat_direction(lat_direction, ys): +def _prepare_lat_direction(lat_direction: bool, ys: slice): # Check direction of latitudes encoded in dataset, flip if necessary if not lat_direction and ys.stop > ys.start: ys = slice(ys.stop, ys.start, -ys.step if ys.step is not None else None) if lat_direction and ys.stop < ys.start: ys = slice(ys.stop, ys.start, ys.step if ys.step is not None else None) + return ys diff --git a/src/geodata/types.py b/src/geodata/types.py index 35fa48ef..70dafba0 100644 --- a/src/geodata/types.py +++ b/src/geodata/types.py @@ -17,4 +17,4 @@ CoordRange = slice | tuple[float, float] | list[float] BoundRange = tuple[float, float, float, float] | list[float] -__all__ = ["DateRange", "CoordRange"] +__all__ = ["DateRange", "CoordRange", "BoundRange"] From c6ad195ed2118d4c2e999c371df85d1b89eab481 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 21 Feb 2025 00:33:25 +0000 Subject: [PATCH 15/54] feat: add wind_solar variables for HRRR --- src/geodata/convert.py | 3 +- src/geodata/datasets/hrrr/_base.py | 4 ++ src/geodata/datasets/hrrr/wind_solar.py | 50 ++++++++++++++++++++++++- src/geodata/preparation.py | 2 + 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/geodata/convert.py b/src/geodata/convert.py index 6a5e0481..bdc86fcb 100644 --- a/src/geodata/convert.py +++ b/src/geodata/convert.py @@ -1,4 +1,5 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) +# Copyright 2025 Xiqiang Liu, Michael Davidson (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -74,7 +75,7 @@ def convert_solar_thermal( output = irradiation * eta - return (output).where(output > 0.0).fillna(0.0) + return output.where(output > 0.0).fillna(0.0) def convert_pv(ds, panel, orientation, trigon_model="simple", clearsky_model="simple"): diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index 2c81127e..f6e8cc27 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -38,6 +38,10 @@ class HRRRBaseDataset(BaseDataset): frequency = "daily" _priority = ["google", "aws", "azure"] + lat_direction = True + meta_prepare_func = None + tasks_func = None + def _extra_setup(self, **kwargs): self._herbie_save_dir = tempfile.TemporaryDirectory() diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/wind_solar.py index 881fbc2d..e4e9b919 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/wind_solar.py @@ -67,11 +67,20 @@ def _download_file(self, file: AtomicDataset): with redirect_stdout_to_logger(logger, logging.INFO): fh.download(":[UV]GRD:[1,8]0 m") fh.download(":TMP:2 m") + fh.download(":..WRF:surface") + fh.download(":TCDC:entire atmosphere") + fh.download(":LCDC:low cloud layer") + fh.download(":MCDC:middle cloud layer") + fh.download(":HCDC:high cloud layer") uv_10 = [] uv_80 = [] tmp_2 = [] wrfs = [] + tcdc = [] + lcdc = [] + mcdc = [] + hcdc = [] for hour in date_range: h = Herbie( @@ -95,6 +104,26 @@ def _download_file(self, file: AtomicDataset): wrfs.append( self._preprocess_individual_herbie(h, ":..WRF:surface", hour) ) + tcdc.append( + self._preprocess_individual_herbie( + h, ":TCDC:entire atmosphere", hour + ) + ) + lcdc.append( + self._preprocess_individual_herbie( + h, ":LCDC:low cloud layer", hour + ) + ) + mcdc.append( + self._preprocess_individual_herbie( + h, ":MCDC:middle cloud layer", hour + ) + ) + hcdc.append( + self._preprocess_individual_herbie( + h, ":HCDC:high cloud layer", hour + ) + ) except ValueError: logger.warning(f"No data found for {hour}, skipping.") @@ -113,8 +142,22 @@ def _download_file(self, file: AtomicDataset): wrfs: xr.Dataset = xr.open_mfdataset( wrfs, concat_dim="time", chunks="auto", combine="nested" ) + tcdc: xr.Dataset = xr.open_mfdataset( + tcdc, concat_dim="time", chunks="auto", combine="nested" + ) + lcdc: xr.Dataset = xr.open_mfdataset( + lcdc, concat_dim="time", chunks="auto", combine="nested" + ) + mcdc: xr.Dataset = xr.open_mfdataset( + mcdc, concat_dim="time", chunks="auto", combine="nested" + ) + hcdc: xr.Dataset = xr.open_mfdataset( + hcdc, concat_dim="time", chunks="auto", combine="nested" + ) - ds = xr.merge([uv_10, uv_80, tmp_2, wrfs], compat="override") + ds = xr.merge( + [uv_10, uv_80, tmp_2, wrfs, lcdc, tcdc, mcdc, hcdc], compat="override" + ) try: del ds["heightAboveGround"] @@ -122,6 +165,11 @@ def _download_file(self, file: AtomicDataset): del ds.attrs["local_grib"] del ds.attrs["remote_grib"] del ds.coords["gribfile_projection"] + del ds.coords["surface"] + del ds.coords["lowCloudLayer"] + del ds.coords["middleCloudLayer"] + del ds.coords["highCloudLayer"] + del ds.coords["atmosphere"] except KeyError: pass diff --git a/src/geodata/preparation.py b/src/geodata/preparation.py index 5226668b..b973ad0b 100644 --- a/src/geodata/preparation.py +++ b/src/geodata/preparation.py @@ -33,6 +33,8 @@ if TYPE_CHECKING: from geodata.cutout import Cutout +else: + Cutout = object logger = logging.getLogger(__name__) From baafcbb290f1817e7cd652ec352d69a595d255b9 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 21 Feb 2025 19:14:03 +0000 Subject: [PATCH 16/54] feat: finalize HRRR 3D wind dataset --- src/geodata/datasets/_base.py | 10 +++- src/geodata/datasets/hrrr/wind_3d.py | 78 +++++++++++++++++++++------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 5edde962..567a479b 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -292,12 +292,18 @@ def download(self, force: bool = False): # Post-process the dataset for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): if file.check(): - ds = xr.open_dataset(file.path) + ds = xr.open_dataset(file.path).chunk() ds = self._rename_and_clean_coords(ds) ds = self._dataset_postprocess(ds) - ds.to_netcdf(file.path) + + # xarray does not support overwriting files, so we must save the + # dataset to a new file and then rename it backwards + ds.to_netcdf(file.path.with_stem(file.path.stem + "_postprocessed")) ds.close() + file.path.unlink() + file.path.with_stem(file.path.stem + "_postprocessed").rename(file.path) + def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): """Method to postprocess the dataset after it has been downloaded. This method should be implemented by subclasses to handle any diff --git a/src/geodata/datasets/hrrr/wind_3d.py b/src/geodata/datasets/hrrr/wind_3d.py index 25220bde..7be26689 100644 --- a/src/geodata/datasets/hrrr/wind_3d.py +++ b/src/geodata/datasets/hrrr/wind_3d.py @@ -15,6 +15,7 @@ import logging import multiprocessing as mp +import os import pandas as pd import xarray as xr @@ -33,15 +34,15 @@ class HRRR3DWindHourlyDataset(HRRRBaseDataset): and storing of these datasets. Variables: - - u_fixed: zonal wind speed at 10m and 80m - - v_fixed: meridional wind speed at 10m and 80m - - wind_speed_fixed: wind speed at 10m and 80m - - u_hybrid: zonal wind speed at hybrid levels (variable across locations) - - v_hybrid: meridional wind speed at hybrid levels (variable across locations) + - u: zonal wind component + - v: meridional wind component + - gh: geopotential height Important Coordinates: - - heightAboveGround: height above ground level of the fixed levels - - hybrid: hybrid level of the hybrid levels (fixed 1,2,3,4) + - time: time of the observation + - level: hybrid pressure level + - y: latitude + - x: longitude """ weather_config = "wind_3d" @@ -102,19 +103,60 @@ def _download_file(self, file: AtomicDataset): except ValueError: logger.warning(f"No data found for {hour}, skipping.") - uv_10 = xr.concat(uv_10, dim="time") - uv_80 = xr.concat(uv_80, dim="time") - uv_hybrid = xr.concat(uv_hybrid, dim="time") - hgt_hybrid = xr.concat(hgt_hybrid, dim="time") + # Offload to disk temporarily to save memory + xr.concat(uv_10, dim="time").to_netcdf( + os.path.join(self._herbie_save_dir.name, "uv_10.nc") + ) + del uv_10 + xr.concat(uv_80, dim="time").to_netcdf( + os.path.join(self._herbie_save_dir.name, "uv_80.nc") + ) + del uv_80 + xr.concat(uv_hybrid, dim="time").to_netcdf( + os.path.join(self._herbie_save_dir.name, "uv_hybrid.nc") + ) + del uv_hybrid + xr.concat(hgt_hybrid, dim="time").to_netcdf( + os.path.join(self._herbie_save_dir.name, "hgt_hybrid.nc") + ) + del hgt_hybrid + + uv_10 = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_10.nc"), chunks="auto" + ) + uv_80 = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_80.nc"), chunks="auto" + ) + uv_hybrid = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_hybrid.nc"), chunks="auto" + ) + hgt_hybrid = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "hgt_hybrid.nc"), chunks="auto" + ) ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") - ds = ds.rename({"u": "u_fixed", "v": "v_fixed"}) - ds["wind_speed_fixed"] = (ds["u_fixed"] ** 2 + ds["u_fixed"] ** 2) ** 0.5 - - uv_hybrid = uv_hybrid.rename({"u": "u_hybrid", "v": "v_hybrid"}) - ds = xr.merge([ds, uv_hybrid, hgt_hybrid]) - - del uv_10, uv_80, uv_hybrid, hgt_hybrid + heights = ds["heightAboveGround"].broadcast_like(ds["u"]) + + ds["u"] = xr.concat( + [ds["u"].rename({"heightAboveGround": "hybrid"}), uv_hybrid["u"]], + dim="hybrid", + ) + ds["v"] = xr.concat( + [ds["v"].rename({"heightAboveGround": "hybrid"}), uv_hybrid["v"]], + dim="hybrid", + ) + del ds["heightAboveGround"] + + ds["gh"] = xr.concat( + [heights.rename({"heightAboveGround": "hybrid"}), hgt_hybrid["gh"]], + dim="hybrid", + ).astype("float32") + ds = ds.rename({"hybrid": "level"}).sortby("level") + + ds["level"].values[-2:] = [-1, -2] + ds["level"] = ds["level"].astype("int8") + + ds = ds.sortby("level") try: del ds.attrs["search"] From 69177c52c9b51c5fc76c47db81e7418608755527 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 28 Feb 2025 02:15:32 +0000 Subject: [PATCH 17/54] fix: make all datasets directories modules --- src/geodata/datasets/era5/__init__.py | 5 +- src/geodata/datasets/era5/hourly/__init__.py | 18 +++ src/geodata/datasets/era5/monthly/__init__.py | 18 +++ src/geodata/datasets/hrrr/__init__.py | 18 +++ src/geodata/datasets/hrrr/hourly/__init__.py | 19 ++++ .../datasets/hrrr/{ => hourly}/wind_3d.py | 6 +- .../datasets/hrrr/{ => hourly}/wind_solar.py | 6 +- src/geodata/datasets/hrrr/wind.py | 107 ------------------ src/geodata/datasets/merra2/__init__.py | 18 +++ .../datasets/merra2/hourly/__init__.py | 18 +++ 10 files changed, 117 insertions(+), 116 deletions(-) create mode 100644 src/geodata/datasets/era5/hourly/__init__.py create mode 100644 src/geodata/datasets/era5/monthly/__init__.py create mode 100644 src/geodata/datasets/hrrr/__init__.py create mode 100644 src/geodata/datasets/hrrr/hourly/__init__.py rename src/geodata/datasets/hrrr/{ => hourly}/wind_3d.py (98%) rename src/geodata/datasets/hrrr/{ => hourly}/wind_solar.py (98%) delete mode 100644 src/geodata/datasets/hrrr/wind.py create mode 100644 src/geodata/datasets/merra2/__init__.py create mode 100644 src/geodata/datasets/merra2/hourly/__init__.py diff --git a/src/geodata/datasets/era5/__init__.py b/src/geodata/datasets/era5/__init__.py index 76d5add4..ca00fb2d 100644 --- a/src/geodata/datasets/era5/__init__.py +++ b/src/geodata/datasets/era5/__init__.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from .hourly.wind_solar import ERA5WindSolarHourlyDataset -from .monthly.wind_solar import ERA5WindSolarMonthlyDataset +from . import hourly, monthly -__all__ = ["ERA5WindSolarHourlyDataset", "ERA5WindSolarMonthlyDataset"] +__all__ = ["hourly", "monthly"] diff --git a/src/geodata/datasets/era5/hourly/__init__.py b/src/geodata/datasets/era5/hourly/__init__.py new file mode 100644 index 00000000..fbaa1656 --- /dev/null +++ b/src/geodata/datasets/era5/hourly/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_solar import ERA5WindSolarHourlyDataset + +__all__ = ["ERA5WindSolarHourlyDataset"] diff --git a/src/geodata/datasets/era5/monthly/__init__.py b/src/geodata/datasets/era5/monthly/__init__.py new file mode 100644 index 00000000..98d2dfe8 --- /dev/null +++ b/src/geodata/datasets/era5/monthly/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_solar import ERA5WindSolarMonthlyDataset + +__all__ = ["ERA5WindSolarMonthlyDataset"] diff --git a/src/geodata/datasets/hrrr/__init__.py b/src/geodata/datasets/hrrr/__init__.py new file mode 100644 index 00000000..334f2004 --- /dev/null +++ b/src/geodata/datasets/hrrr/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import hourly + +__all__ = ["hourly"] diff --git a/src/geodata/datasets/hrrr/hourly/__init__.py b/src/geodata/datasets/hrrr/hourly/__init__.py new file mode 100644 index 00000000..5b6188c6 --- /dev/null +++ b/src/geodata/datasets/hrrr/hourly/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_3d import HRRR3DWindHourlyDataset +from .wind_solar import HRRRHourlyDataset + +__all__ = ["HRRRHourlyDataset", "HRRR3DWindHourlyDataset"] diff --git a/src/geodata/datasets/hrrr/wind_3d.py b/src/geodata/datasets/hrrr/hourly/wind_3d.py similarity index 98% rename from src/geodata/datasets/hrrr/wind_3d.py rename to src/geodata/datasets/hrrr/hourly/wind_3d.py index 7be26689..4310ac9c 100644 --- a/src/geodata/datasets/hrrr/wind_3d.py +++ b/src/geodata/datasets/hrrr/hourly/wind_3d.py @@ -21,9 +21,9 @@ import xarray as xr from herbie import FastHerbie, Herbie -from ...logging import redirect_stdout_to_logger -from .._base import AtomicDataset -from ._base import HRRRBaseDataset +from ....logging import redirect_stdout_to_logger +from ..._base import AtomicDataset +from .._base import HRRRBaseDataset logger = logging.getLogger(__name__) diff --git a/src/geodata/datasets/hrrr/wind_solar.py b/src/geodata/datasets/hrrr/hourly/wind_solar.py similarity index 98% rename from src/geodata/datasets/hrrr/wind_solar.py rename to src/geodata/datasets/hrrr/hourly/wind_solar.py index e4e9b919..5252fc14 100644 --- a/src/geodata/datasets/hrrr/wind_solar.py +++ b/src/geodata/datasets/hrrr/hourly/wind_solar.py @@ -20,9 +20,9 @@ import xarray as xr from herbie import FastHerbie, Herbie -from ...logging import redirect_stdout_to_logger -from .._base import AtomicDataset -from ._base import HRRRBaseDataset +from ....logging import redirect_stdout_to_logger +from ..._base import AtomicDataset +from .._base import HRRRBaseDataset logger = logging.getLogger(__name__) diff --git a/src/geodata/datasets/hrrr/wind.py b/src/geodata/datasets/hrrr/wind.py deleted file mode 100644 index 6cefab69..00000000 --- a/src/geodata/datasets/hrrr/wind.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -import multiprocessing as mp - -import pandas as pd -import xarray as xr -from herbie import FastHerbie, Herbie - -from ...logging import redirect_stdout_to_logger -from .._base import AtomicDataset -from ._base import HRRRBaseDataset - -logger = logging.getLogger(__name__) - - -class HRRRWindHourlyDataset(HRRRBaseDataset): - """HRRRWindHourlyDataset is a class that encaps a dataset from the HRRR - dataset. It provides a streamlined workflow for downloading, preprocessing, - and storing of these datasets. - - The HRRR dataset is a high-resolution weather forecast model that provides - hourly data for the United States. This dataset is useful for a variety of - applications, including renewable energy forecasting, weather prediction, - and climate research. - - This class provides a simple interface for downloading and processing the - HRRR dataset. It allows users to specify the years and months of interest, - as well as the variables they wish to download. - """ - - weather_config = "wind" - product = "sfc" - - def _download_file(self, file: AtomicDataset): - year, month, day = file.year, file.month, file.day - - date_range = pd.date_range( - f"{year}-{month}-{day}", - f"{year}-{month}-{day+1}", - freq="h", - inclusive="left", - ) - - fh = FastHerbie( - date_range, - model=self.module, - product=self.product, - max_threads=mp.cpu_count() * 2, - save_dir=self._herbie_save_dir.name, - priority=self._priority, - ) - - with redirect_stdout_to_logger(logger, logging.INFO): - logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") - fh.download(":[UV]GRD:[1,8]0 m") - - uv_10 = [] - uv_80 = [] - for hour in date_range: - h = Herbie( - hour, - model=self.module, - product=self.product, - save_dir=self._herbie_save_dir.name, - priority=self._priority, - ) - - try: - uv_10.append( - h.xarray("[UV]GRD:10 m").rename({"u10": "u", "v10": "v"}) - ) - uv_80.append(h.xarray("[UV]GRD:80 m")) - except ValueError: - logger.warning(f"No data found for {hour}, skipping.") - - uv_10 = xr.concat(uv_10, dim="time") - uv_80 = xr.concat(uv_80, dim="time") - - ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") - ds["wind_speed"] = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 - - try: - del ds.attrs["search"] - del ds.attrs["local_grib"] - del ds.attrs["remote_grib"] - except KeyError: - pass - - ds.to_netcdf(file.path) - - # NOTE: Flush temporary FastHerbie save directory to save space, since we no - # longer need the raw downloaded files - self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/merra2/__init__.py b/src/geodata/datasets/merra2/__init__.py new file mode 100644 index 00000000..334f2004 --- /dev/null +++ b/src/geodata/datasets/merra2/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import hourly + +__all__ = ["hourly"] diff --git a/src/geodata/datasets/merra2/hourly/__init__.py b/src/geodata/datasets/merra2/hourly/__init__.py new file mode 100644 index 00000000..cd3c4962 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .surface_flux import MERRA2SurfaceFluxHourlyDataset + +__all__ = ["MERRA2SurfaceFluxHourlyDataset"] From bbcdbf4a1dc0798904efb53790370ccbec09e381 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 3 Mar 2025 01:24:33 +0000 Subject: [PATCH 18/54] misc: remove progressbar as a dependency of the package --- pyproject.toml | 1 - uv.lock | 34 +++++----------------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 24a34871..d7532949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "rasterio==1.4.0", "rioxarray==0.14.0", "shapely>=2.0.6", - "progressbar2>=4.5.0", "geopandas>=1.0.1", "pyyaml>=6.0.2", "dask>=2024.9.0", diff --git a/uv.lock b/uv.lock index 0cf79794..66a8ec76 100644 --- a/uv.lock +++ b/uv.lock @@ -1,4 +1,5 @@ version = 1 +revision = 1 requires-python = ">=3.10" resolution-markers = [ "python_full_version < '3.11'", @@ -458,7 +459,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } wheels = [ @@ -803,7 +804,6 @@ dependencies = [ { name = "numexpr" }, { name = "numpy" }, { name = "pandas" }, - { name = "progressbar2" }, { name = "pyproj" }, { name = "pyyaml" }, { name = "rasterio" }, @@ -858,7 +858,6 @@ requires-dist = [ { name = "numexpr", specifier = "==2.10.1" }, { name = "numpy", specifier = "<2" }, { name = "pandas", specifier = ">=2.2.3" }, - { name = "progressbar2", specifier = ">=4.5.0" }, { name = "pyproj", specifier = "==3.6.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rasterio", specifier = "==1.4.0" }, @@ -873,6 +872,7 @@ requires-dist = [ { name = "tqdm", specifier = ">=4.66.5" }, { name = "xarray", specifier = ">=2024.9.0" }, ] +provides-extras = ["download", "notebook", "docs", "accelerate"] [package.metadata.requires-dev] dev = [ @@ -1050,7 +1050,7 @@ name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "platform_system == 'Darwin'" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, { name = "comm" }, { name = "debugpy" }, { name = "ipython" }, @@ -2102,18 +2102,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, ] -[[package]] -name = "progressbar2" -version = "4.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-utils" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/24/3587e795fc590611434e4bcb9fbe0c3dddb5754ce1a20edfd86c587c0004/progressbar2-4.5.0.tar.gz", hash = "sha256:6662cb624886ed31eb94daf61e27583b5144ebc7383a17bae076f8f4f59088fb", size = 101449 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/94/448f037fb0ffd0e8a63b625cf9f5b13494b88d15573a987be8aaa735579d/progressbar2-4.5.0-py3-none-any.whl", hash = "sha256:625c94a54e63915b3959355e6d4aacd63a00219e5f3e2b12181b76867bf6f628", size = 57132 }, -] - [[package]] name = "prometheus-client" version = "0.21.0" @@ -2343,18 +2331,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/a6/145655273568ee78a581e734cf35beb9e33a370b29c5d3c8fee3744de29f/python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd", size = 8067 }, ] -[[package]] -name = "python-utils" -version = "3.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/99/fd1e3f80357dd88378281013ae7040a443de395bb0855bf17cbb828488d1/python_utils-3.9.0.tar.gz", hash = "sha256:3689556884e3ae53aec5a4c9f17b36e752a3e93a7ba2768c6553fc4dd6fa70ef", size = 35352 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/e7/200045de1f41ad74915a334f7c2aac54d2d6f4b89643a53e1062cde4895b/python_utils-3.9.0-py2.py3-none-any.whl", hash = "sha256:a7719a5ef4bae7360d2a15c13b08c4e3c3e39b9df19bd16f119ff8d0cfeaafb7", size = 32085 }, -] - [[package]] name = "pytz" version = "2024.2" @@ -3101,7 +3077,7 @@ name = "tqdm" version = "4.66.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/83/6ba9844a41128c62e810fddddd72473201f3eacde02046066142a2d96cc5/tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad", size = 169504 } wheels = [ From 31e22c01420d60d51610fbb652b1c37fd9ea6fee Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:16:59 +0000 Subject: [PATCH 19/54] feat: cutout's preparation is now working with the new dataset --- src/geodata/cutout.py | 87 ++++++++-------- src/geodata/datasets/_base.py | 68 +++++++++---- src/geodata/datasets/era5/_base.py | 86 ++++++++++++++++ src/geodata/datasets/hrrr/_base.py | 68 ++++++++++++- src/geodata/datasets/merra2/_base.py | 144 +++++++++++++++++++-------- src/geodata/preparation.py | 28 ++---- src/geodata/utils.py | 44 ++++---- 7 files changed, 371 insertions(+), 154 deletions(-) diff --git a/src/geodata/cutout.py b/src/geodata/cutout.py index 1f020e6e..bae85889 100644 --- a/src/geodata/cutout.py +++ b/src/geodata/cutout.py @@ -23,7 +23,7 @@ from functools import partial from operator import itemgetter from pathlib import Path -from typing import Literal, Optional, Sequence, Union +from typing import Literal, Optional, Union import numpy as np import pyproj @@ -51,11 +51,9 @@ cutout_prepare, cutout_produce_specific_dataseries, ) -from .resource import ( - get_solarpanelconfig, - get_windturbineconfig, - windturbine_smooth, -) +from .resource import get_solarpanelconfig, get_windturbineconfig, windturbine_smooth +from .types import BoundRange, DateRange +from .utils import ensure_slice logger = logging.getLogger(__name__) @@ -86,12 +84,12 @@ def __init__( self, name: str, dataset_cls: type[BaseDataset], - years: slice, + years: DateRange, cutout_dir: Union[str, Path] = config.cutout_dir, - bounds: Optional[Sequence] = None, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, + bounds: Optional[BoundRange] = None, + months: Optional[DateRange] = None, + xs: Optional[BoundRange] = None, + ys: Optional[BoundRange] = None, ): self.name = name self.cutout_dir = Path(cutout_dir, name) @@ -106,12 +104,14 @@ def __init__( self.area = None params_dict = { - "years": years, + "years": ensure_slice(years), "months": months, "xs": xs, "ys": ys, } + if bounds is None and (xs is None and ys is None): + raise TypeError("Either bounds or xs/ys arguments must be specified.") if bounds is not None and (xs is not None or ys is not None): raise TypeError("Cannot specify both bounds and xs/ys arguments.") @@ -123,16 +123,15 @@ def __init__( if months is None: logger.info("No months specified, defaulting to 1-12") params_dict.update(months=slice(1, 12)) + params_dict["months"] = ensure_slice(months) - if self.cutout_dir.is_dir(): - # Check if cutout directory exists - # If cutout dir exists, check completness of files - if self.meta_path.is_file(): # open existing meta file - self.meta = xr.open_dataset(self.get_filename()).stack( - dim={"year-month": ("year", "month")} - ) + self.prepared = False + if self.cutout_dir.is_dir() and self.meta_path.is_file(): + self.meta = xr.open_dataset(self._get_filename()).stack( + dim={"year-month": ("year", "month")} + ) - if all(self.catalog): + if all(f.is_file() for f in self.catalog): logger.info("All cutout (%s, %s) files available.", name, cutout_dir) # At least one of xs, ys is in params_dict @@ -143,40 +142,36 @@ def __init__( # Subset is available self.prepared = True logger.info("Cutout subset prepared: %s", self) + return else: logger.info("Cutout subset not available: %s", self) else: # No subsetting of bounds. Keep full cutout self.prepared = True logger.info("Cutout prepared: %s", self) + return - else: - # Not all files accounted for - self.prepared = False - logger.info("Cutout (%s, %s) not complete.", name, cutout_dir) - - # In case - if not self.prepared: - logger.info("Cutout (%s, %s) not found or incomplete.", name, cutout_dir) - - if {"xs", "ys", "years"}.difference(params_dict): - raise TypeError( - "Arguments `xs`, `ys`, and `years` need to be specified to create " - "a cutout." - ) + logger.info("Cutout (%s, %s) not complete.", name, cutout_dir) + if {"xs", "ys", "years"}.difference(params_dict): + raise TypeError( + "Arguments `xs`, `ys`, and `years` need to be specified to create " + "a cutout." + ) - if self.meta is not None: - # if meta.nc exists, close and delete it - self.meta.close() - self.meta_path.unlink() + if self.meta is not None: + # if meta.nc exists, close and delete it + self.meta.close() + self.meta_path.unlink() - self.meta = self.get_meta(**params_dict) - self.cutout_dir.mkdir(parents=True, exist_ok=True) + self.meta = self.get_meta(**params_dict) + self.cutout_dir.mkdir(parents=True, exist_ok=True) - # Write meta file - self.meta_clean.unstack("year-month").to_netcdf(self.meta_path) + # Write meta file + self.meta_clean.unstack("year-month").to_netcdf(self.meta_path) - def _get_filename(self, year: int | None = None, month: int | None = None): + def _get_filename( + self, year: int | tuple[int, int] | None = None, month: int | None = None + ): """Return path to dataset xarray files related to this Cutout. If both year and month are None, return path to meta.nc file. @@ -192,10 +187,10 @@ def _get_filename(self, year: int | None = None, month: int | None = None): return self.cutout_dir / "meta.nc" if year is not None: - if month is not None: - return self.cutout_dir / f"{year}{month:0>2}.nc" + if month is None: + year, month = year - return self.cutout_dir / f"{year}.nc" + return self.cutout_dir / f"{year}{month:0>2}.nc" @property def catalog(self): diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 567a479b..b861f57c 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -20,13 +20,14 @@ import logging from collections.abc import Sequence from pathlib import Path +from typing import Literal import pandas as pd import xarray as xr from tqdm.auto import tqdm from ..config import DATASET_ROOT_PATH -from ..types import BoundRange, DateRange +from ..types import BoundRange, CoordRange, DateRange logger = logging.getLogger(__name__) @@ -136,7 +137,7 @@ class BaseDataset(abc.ABC): module: str weather_config: str - frequency: str = "monthly" + frequency: Literal["hourly", "daily", "monthly"] = "monthly" def __init__( self, @@ -196,11 +197,7 @@ def __init__( raise ValueError("Latitude bounds must be between -90 and 90") self.bounds = bounds - self.storage_root = ( - Path(kwargs.get("dataset_root", DATASET_ROOT_PATH)) - / self.module - / self.weather_config - ) + self.storage_root = DATASET_ROOT_PATH / self.module / self.weather_config if not self.storage_root.exists(): logger.info( f"Storage directory for {self.__class__.__name__} does not exist, " @@ -253,18 +250,12 @@ def downloaded(self): return all((file.check() for file in self.catalog)) @abc.abstractmethod - def _download_file(self, file: dict): + def _download_file(self, file: AtomicDataset): """Method to download a single file from the dataset. This method should download the file and save it to the appropriate location. Args: - file: A dictionary containing the metadata of the file to download. At the - minimum, this dictionary should contain the following keys: - - year: the year of the file - - month: the month of the file - - day: the day of the file (if applicable) - - hour: the hour of the file (if applicable) - - save_path: the path where the file should be saved + file: An instance of AtomicDataset representing the file to download. """ def download(self, force: bool = False): @@ -392,13 +383,27 @@ def submodule(self): @staticmethod @abc.abstractmethod - def tasks_func(): + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + prepare_func, + **meta_attrs, + ): """A method that returns a list of tasks that can be run on the dataset.""" @staticmethod @abc.abstractmethod - def meta_prepare_func(): - """A method that returns a list of metadata preparation tasks.""" + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + """A method that generates the metadata for the cutout.""" + + @staticmethod + @abc.abstractmethod + def prepare_func( + fn: str | Path, year: int, month: int, xs: CoordRange, ys: CoordRange, **kwargs + ): + """A method that prepares the cutout for individual dataset.""" @property def catalog(self) -> list["AtomicDataset"]: @@ -476,7 +481,34 @@ def _rename_and_clean_coords(self, ds: xr.Dataset, add_lon_lat: bool = False): if "lon" in ds.coords: ds = ds.rename({"lon": "x"}) + # Flatten x and y if they are multi-dimensional + if ds.coords["x"].ndim > 1: + ds = ds.assign_coords(x=("x", ds.coords["x"][0].values)) + if ds.coords["y"].ndim > 1: + ds = ds.assign_coords(y=("y", ds.coords["y"][:, 0].values)) + if add_lon_lat: ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) return ds + + @classmethod + def _get_files(cls, year: int, month: int): + """Get the path where the file should be saved. Internal method used by Cutouts. + + Args: + year: The year of the file. + month: The month of the file. + """ + + storage_root = DATASET_ROOT_PATH / cls.module / cls.weather_config + + match cls.frequency: + case "monthly": + return [storage_root / str(year) / f"{month:02d}.nc"] + case "daily": + return list((storage_root / str(year) / f"{month:02d}").glob("*.nc")) + case _: + raise ValueError( + f"Invalid frequency {cls.frequency} defined for this dataset." + ) diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index 670af3d4..7b5824f8 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -16,9 +16,51 @@ import logging import cdsapi +import numpy as np +import xarray as xr +from ...types import CoordRange from .._base import BaseDataset +logger = logging.getLogger(__name__) + + +def _convert_and_subset_lons_lats_era5(ds: xr.Dataset, xs: slice, ys: slice): + # Rename geographic dimensions to x,y + # Subset x,y according to xs, ys (subset_x_y_era5) + + # Longitudes should go from -180. to +180. + if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): + ds = xr.concat( + [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" + ) + ds = ds.assign_coords( + x=np.where( + ds.coords["x"].values <= 180, + ds.coords["x"].values, + ds.coords["x"].values - 360.0, + ) + ) + + # Subset x and y + return _subset_x_y_era5(ds, xs, ys) + + +def _subset_x_y_era5(ds: xr.Dataset, xs: slice, ys: slice): + # Subset x,y according to xs, ys + + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + ds = ds.sel(x=xs) + + return ds + class ERA5BaseDataset(BaseDataset): """ERA5BaseDataset is a class that encaps a dataset from the ERA5 reanalysis @@ -38,3 +80,47 @@ def _extra_setup(self, **kwargs): debug_callback=self.logger.debug, warning_callback=self.logger.warning, ) + + @classmethod + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + # Reference of the quantities + # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation + # Geopotential is aka Orography in the CDS: + # https://confluence.ecmwf.int/pages/viewpage.action?pageId=78296105 + + with xr.open_mfdataset(cls._get_path(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _convert_and_subset_lons_lats_era5(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + prepare_func: callable, + **meta_attrs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + fn = meta_attrs["fn"] + + logger.info(yearmonths) + logger.info(list(yearmonths)) + + return [ + dict( + prepare_func=prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format(year=year, month=month), + ) + for year, month in yearmonths + ] diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index f6e8cc27..4726b680 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -19,14 +19,32 @@ import tempfile import herbie +import numpy as np import pandas as pd import xarray as xr +from ...types import CoordRange from .._base import BaseDataset logger = logging.getLogger(__name__) +def _subset_x_y(ds: xr.Dataset, xs: slice, ys: slice): + # Subset x,y according to xs, ys + + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + ds = ds.sel(x=xs) + + return ds + + class HRRRBaseDataset(BaseDataset): """HRRRBaseDataset is a class that encaps a dataset from the HRRR dataset. It provides a streamlined workflow for downloading, preprocessing, @@ -39,8 +57,6 @@ class HRRRBaseDataset(BaseDataset): _priority = ["google", "aws", "azure"] lat_direction = True - meta_prepare_func = None - tasks_func = None def _extra_setup(self, **kwargs): self._herbie_save_dir = tempfile.TemporaryDirectory() @@ -69,4 +85,52 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): logger.debug("Fixing longitude range") ds["x"] = (ds["x"] % 360 + 540) % 360 - 180 + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + return ds + + @classmethod + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + with xr.open_mfdataset(cls._get_files(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _subset_x_y(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + **kwargs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + + return [ + dict( + prepare_func=cls.prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=cls._get_files(year, month, **kwargs), + ) + for year, month in yearmonths + ] + + @staticmethod + def prepare_func(fn, year, month, xs, ys, **kwargs): + if isinstance(fn, str) and not osp.exists(fn): + return + if isinstance(fn, list) and not all(osp.isfile(f) for f in fn): + return + + with xr.open_mfdataset(fn) as ds: + logger.info("Opening %s", fn) + + yield (year, month), _subset_x_y(ds, xs, ys) diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index e3492d15..d09a1050 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging +from calendar import monthrange import numpy as np import requests @@ -21,6 +23,38 @@ from ...types import CoordRange from .._base import AtomicDataset, BaseDataset +logger = logging.getLogger(__name__) + + +def _convert_and_subset_lons_lats_merra2( + ds: xr.Dataset, xs: CoordRange, ys: CoordRange +): + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + + # Lons should go from -180. to +180. + if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): + ds = xr.concat( + [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" + ) + ds = ds.assign_coords( + lon=np.where( + ds.coords["x"].values <= 180, + ds.coords["x"].values, + ds.coords["x"].values - 360.0, + ) + ) + else: + ds = ds.sel(x=xs) + + return ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + class MERRA2BaseDataset(BaseDataset): """MERRA2BaseDataset is a class that encaps a dataset from the MERRA2 reanalysis @@ -76,47 +110,6 @@ def spinup_year(self, year: int, month: int): return spinup - def convert_and_subset_lons_lats_merra2( - ds: xr.Dataset | xr.DataArray, xs: CoordRange, ys: CoordRange - ) -> xr.Dataset | xr.DataArray: - """Rename geographic dimensions to x,y. Subset x,y according to xs, ys. - - Args: - ds (xr.Dataset | xr.DataArray): The dataset to subset - xs (slice): The slice of longitudes to subset - ys (slice): The slice of latitudes to subset - - Returns: - xr.Dataset | xr.DataArray: The subsetted dataset - """ - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(lat=ys) - - # Longitudes should go from -180. to +180. - if len(ds.coords["lon"].sel(lon=slice(xs.start + 360.0, xs.stop + 360.0))): - ds = xr.concat( - [ds.sel(lon=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(lon=xs)], - dim="lon", - ) - ds = ds.assign_coords( - lon=np.where( - ds.coords["lon"].values <= 180, - ds.coords["lon"].values, - ds.coords["lon"].values - 360.0, - ) - ) - else: - ds = ds.sel(lon=xs) - - return super()._rename_and_clean_coords(ds) - def _daily_catalog(self): if not self.url_template: raise NotImplementedError("url_template is not defined for this dataset") @@ -128,3 +121,72 @@ def _daily_catalog(self): file.url = self.url_template.format(**vars(file)) return catalog + + @classmethod + def meta_prepare_func( + cls, xs: CoordRange, ys: CoordRange, year: int, month: int, **params + ): + with xr.open_mfdataset(cls._get_files(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _convert_and_subset_lons_lats_merra2(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + prepare_func: callable, + **meta_attrs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + fn = meta_attrs["fn"] + + match cls.frequency: + case "daily": + logger.info(yearmonths) + logger.info( + [ + (year, month, day) + for year, month in yearmonths + for day in range(1, monthrange(year, month)[1] + 1, 1) + ] + ) + + return [ + dict( + prepare_func=prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format( + year=year, + month=month, + day=day, + spinup=cls.spinup_year(year, month), + ), + ) + for year, month in yearmonths + for day in range(1, monthrange(year, month)[1] + 1, 1) + ] + case "monthly": + return [ + dict( + prepare_func=prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format(year=year, month=month), + ) + for year, month in yearmonths + ] + case _: + raise NotImplementedError("Frequency not supported") diff --git a/src/geodata/preparation.py b/src/geodata/preparation.py index b973ad0b..d6fa0240 100644 --- a/src/geodata/preparation.py +++ b/src/geodata/preparation.py @@ -139,29 +139,15 @@ def cutout_prepare( def datasetfn_with_id(ym): # returns a filename with incrementing id at end eg `201101-01.nc` filename = cutout._get_filename(ym) - - base = filename.stem - ext = filename.suffix - - return f"{base}-{i}{ext}" + return filename.with_stem(f"{filename.stem}-{i:02d}") t["filenames"] = {ym: datasetfn_with_id(ym) for ym in yearmonths.tolist()} - # TODO: Using multiple processes will cause issues with geodata currently - # because geodata write the metadata dataframe to the cutout directory - # in every instantiation of a new Cutout object in an unprepared state. - # By running multiple processes, the metadata file will be overwritten - # and there are no locking mechanism currently in place. - # As the result, we will run the tasks in a single process for now. - # In the future, we can consider doing it in a multiprocessing scheme as long as - # we can ensure that the metadata file is being processed in places other than - # the constructor. - - # logger.info( - # "%d tasks have been collected. Starting running them on %s.", - # len(tasks), - # f"{nprocesses} processes" if nprocesses is not None else "all processors", - # ) + logger.info( + "%d tasks have been collected. Starting running them on %s.", + len(tasks), + f"{nprocesses} processes" if nprocesses is not None else "all processors", + ) with Pool(processes=nprocesses) as pool: try: @@ -180,7 +166,7 @@ def datasetfn_with_id(ym): for fn in [cutout._get_filename(ym) for ym in yearmonths.tolist()]: # Find all files with yearmonth prefix eg `201101-XX.nc` - fns = list(cutout_dir.rglob(f"{fn.stem}-*.{fn.suffix}")) + fns = list(cutout_dir.rglob(f"{fn.stem}-*{fn.suffix}")) if len(fns) == 1 and not gebco_height: # Just a single file. Simply rename diff --git a/src/geodata/utils.py b/src/geodata/utils.py index 21b362e4..42640631 100644 --- a/src/geodata/utils.py +++ b/src/geodata/utils.py @@ -18,31 +18,6 @@ import numpy as np import pandas as pd -import progressbar as pgb - - -def make_optional_progressbar(show, prefix, max_value): - if show: - widgets = [ - pgb.widgets.Percentage(), - " ", - pgb.widgets.SimpleProgress(), - " ", - pgb.widgets.Bar(), - " ", - pgb.widgets.Timer(), - " ", - pgb.widgets.ETA(), - ] - if not prefix.endswith(": "): - prefix = prefix.strip() + ": " - maybe_progressbar = pgb.ProgressBar( - prefix=prefix, widgets=widgets, max_value=max_value - ) - else: - maybe_progressbar = lambda x: x # noqa: E731 - - return maybe_progressbar def dummy_njit(f=None, *args, **kwargs): @@ -69,7 +44,7 @@ def get_daterange(years: slice, months: slice): months (slice): The months range. Returns: - pd. + pd.DatetimeIndex: The date range. """ assert years.start <= years.stop, "Start year must be less than stop year." @@ -95,3 +70,20 @@ def default(self, obj): if isinstance(obj, np.ndarray): return obj.tolist() return super(NpEncoder, self).default(obj) + + +def ensure_slice(obj: slice | list): + """Ensure that the input is a slice object. If the input is a list, convert it to a slice object. + + Args: + obj (slice | list): The input object. + + Returns: + slice: The converted slice object. + """ + if isinstance(obj, list) and (len(obj) == 2 or len(obj) == 3): + return slice(*obj) + elif isinstance(obj, slice): + return obj + else: + raise TypeError("Input must be a slice or a list.") From 659566b7bb35b0f7c73e3199f961c56d56a24dc2 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 4 Mar 2025 04:46:30 +0000 Subject: [PATCH 20/54] fix: proper preparation check and refactorization of conversion methods --- src/geodata/convert.py | 344 ++++++++++++++++++++++++++++++++++++ src/geodata/cutout.py | 346 +++---------------------------------- src/geodata/preparation.py | 2 +- 3 files changed, 369 insertions(+), 323 deletions(-) diff --git a/src/geodata/convert.py b/src/geodata/convert.py index bdc86fcb..2dfaf5c3 100644 --- a/src/geodata/convert.py +++ b/src/geodata/convert.py @@ -22,6 +22,7 @@ import datetime as dt import logging from operator import itemgetter +from typing import TYPE_CHECKING, Literal, Callable import numpy as np import xarray as xr @@ -31,6 +32,16 @@ from .pv.orientation import SurfaceOrientation, get_orientation # noqa: F401 from .pv.solar_panel_model import SolarPanelModel from .pv.solar_position import SolarPosition +from .resource import ( + get_solarpanelconfig, + get_windturbineconfig, + windturbine_smooth, +) + +if TYPE_CHECKING: + from .cutout import Cutout +else: + Cutout = object logger = logging.getLogger(__name__) @@ -194,3 +205,336 @@ def convert_pm25(ds): ) return 1e9 * ds["pm25"] # kg / m3 to ug / m3 + + +def heat_demand( + cutout: Cutout, + threshold: float = 15.0, + a: float = 1.0, + constant: float = 0.0, + hour_shift: float = 0.0, + **params, +): + """Convert outside temperature into daily heat demand using the + degree-day approximation. + + Since "daily average temperature" means different things in + different time zones and since xarray coordinates do not handle + time zones gracefully like pd.DateTimeIndex, you can provide an + hour_shift to redefine when the day starts. + + E.g. for Moscow in winter, hour_shift = 4, for New York in winter, + hour_shift = -5 + + This time shift applies across the entire spatial scope of ds for + all times. More fine-grained control will be built in a some + point, i.e. space- and time-dependent time zones. + + WARNING: Because the original data is provided every month, at the + month boundaries there is untidiness if you use a time shift. The + resulting xarray will have duplicates in the index for the parts + of the day in each month at the boundary. You will have to + re-average these based on the number of hours in each month for + the duplicated day. + + Args: + threshold (float): Outside temperature in degrees Celsius above which there is no heat demand. + a (float): Linear factor relating heat demand to outside temperature. + constant (float): Constant part of heat demand that does not depend on outside + temperature (e.g. due to water heating). + hour_shift (float): Time shift relative to UTC for taking daily average + + Returns: + xr.DataArray: Heat demand + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + """ + + return cutout._convert_cutout( + convert_func=convert_heat_demand, + threshold=threshold, + a=a, + constant=constant, + hour_shift=hour_shift, + **params, + ) + + +def temperature(cutout: Cutout, **convert_params): + """Convert temperature in Cutout to outside temperature. + + Args: + convert_params: Keyword arguments passed to `convert_cutout` function + + Returns: + xr.DataArray: Data of the Cutout with temperature converted to outside temperatures. + """ + return cutout._convert_cutout( + convert_func=lambda ds: ds["temperature"] - 273.15, **convert_params + ) + + +def soil_temperature(cutout: Cutout, **convert_params): + """Return soil temperature (useful for e.g. heat pump T-dependent + coefficient of performance). + + Args: + convert_params: Keyword arguments passed to `convert_cutout` function + + Returns: + xr.DataArray: Data of the Cutout with temperature converted to soil temperatures. + """ + return cutout._convert_cutout( + convert_func=lambda ds: (ds["soil temperature"] - 273.15).fillna(0.0), + **convert_params, + ) + + +def solar_thermal( + cutout: Cutout, + orientation: dict | str | Callable | None = None, + trigon_model: str = "simple", + clearsky_model: Literal["simple", "enhanced"] = "simple", + c0: float = 0.8, + c1: float = 3.0, + t_store: float = 80.0, + **params, +): + """Convert downward short-wave radiation flux and outside temperature + into time series for solar thermal collectors. + + Mathematical model and defaults for c0, c1 based on model in [1]. + + Args: + orientation (Union[dict, str, callable]): Panel orientation with slope and azimuth + (units of degrees), or 'latitude_optimal'. + trigon_model (str): Type of trigonometry model + clearsky_model (str): Type of clearsky model for diffuse irradiation. Either + `simple` or `enhanced`. + c0 (float): Parameter for model in [1] This defaults to 0.8. + c1 (float): Parameter for model in [1] This defaults to 3.0. + t_store (float): Store temperature in degree Celsius + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + + References: + [1] Henning and Palzer, Renewable and Sustainable Energy Reviews 30 + (2014) 1003-1018 + """ + + if orientation is None: + orientation = {"slope": 45.0, "azimuth": 180.0} + + if not callable(orientation): + orientation = get_orientation(orientation) + + return cutout._convert_cutout( + convert_func=convert_solar_thermal, + orientation=orientation, + trigon_model=trigon_model, + clearsky_model=clearsky_model, + c0=c0, + c1=c1, + t_store=t_store, + **params, + ) + + +def wind( + cutout: Cutout, + turbine: str | dict, + method: Literal["simple", "interpolation", "extrapolation"], + smooth: bool | dict = False, + **params, +): + """Convert wind speed time-series into wind generation time-series. + + Args: + turbine (Union[str, dict]): Name of a turbine or a dictionary with the parameters + for the wind turbine in [2]. + smooth (Union[bool, dict]): If True, the wind speed time-series will be smoothed + before conversion. If False, no smoothing will be applied. If a dictionary is + passed, the smoothing parameters will be used. + **params: Keyword arguments passed to `convert_cutout` function + """ + + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + + if smooth: + turbine = windturbine_smooth(turbine, params=smooth) + + match method: + case "simple": + return cutout._convert_cutout( + convert_func=convert_wind, turbine=turbine, **params + ) + + case _: + raise ValueError(f"Method {method} not supported.") + + +def windspd(cutout: Cutout, **params): + """ + Generate wind speed time-series + + convert.convert_cutout → convert.convert_windspd + + Parameters + ---------- + **params + Must have 1 of: + turbine : str or dict + Name of a turbine + hub_height : num + Extrapolation height + + Can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + e.g. var_height='lml' + + """ + + if "turbine" in params: + turbine = params.pop("turbine") + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + else: + raise ValueError(f"Turbine ({turbine}) not found.") + hub_height = itemgetter("hub_height")(turbine) + elif "hub_height" in params: + hub_height = params.pop("hub_height") + elif "to_height" in params: + hub_height = params.pop("to_height") + else: + raise ValueError("Either a turbine or hub_height must be specified.") + + params["hub_height"] = hub_height + + return cutout._convert_cutout(convert_func=convert_windspd, **params) + + +def windwpd(cutout: Cutout, **params): + """ + Generate wind power density time-series + + convert.convert_cutout → convert.convert_windwpd + + Parameters + ---------- + **params + Must have 1 of: + turbine : str or dict + Name of a turbine + hub_height : num + Extrapolation height + + Can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + e.g. var_height='lml' + + """ + + if "turbine" in params: + turbine = params.pop("turbine") + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + else: + raise ValueError(f"Turbine ({turbine}) not found.") + hub_height = itemgetter("hub_height")(turbine) + elif "hub_height" in params: + hub_height = params.pop("hub_height") + elif "to_height" in params: + hub_height = params.pop("to_height") + else: + raise ValueError("Either a turbine or hub_height must be specified.") + + params["hub_height"] = hub_height + + return cutout._convert_cutout(convert_func=convert_windwpd, **params) + + +def pv( + cutout: Cutout, + panel: str | dict, + orientation: str | dict | Callable, + clearsky_model: str | None = None, + **params, +): + """Convert downward-shortwave, upward-shortwave radiation flux and + ambient temperature into a pv generation time-series. + + Args: + panel (Union[str, dict]): Panel name known to the reatlas client or a panel config + dictionary with the parameters for the electrical model in [3]. + orientation (Union[str, dict, callback]): Panel orientation can be chosen from either + 'latitude_optimal', a constant orientation {'slope': 0.0, + 'azimuth': 0.0} or a callback function with the same signature + as the callbacks generated by the + `geodata.pv.orientation.make_*` functions. + clearsky_model (Optional[str]): Either the 'simple' or the 'enhanced' Reindl clearsky + model. The default choice of None will choose dependending on + data availability, since the 'enhanced' model also + incorporates ambient air temperature and relative humidity. + + Returns: + xr.DataArray: Time-series or capacity factors based on additional general + conversion arguments. + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + + References: + [1] Soteris A. Kalogirou. Solar Energy Engineering: Processes and Systems, + pages 49-117,469-516. Academic Press, 2009. ISBN 0123745012. + [2] D.T. Reindl, W.A. Beckman, and J.A. Duffie. Diffuse fraction correla- + tions. Solar Energy, 45(1):1 - 7, 1990. + [3] Hans Georg Beyer, Gerd Heilscher and Stefan Bofinger. A Robust Model + for the MPP Performance of Different Types of PV-Modules Applied for + the Performance Check of Grid Connected Systems, Freiburg, June 2004. + Eurosun (ISES Europe Solar Congress). + """ + + if isinstance(panel, str): + panel = get_solarpanelconfig(panel) + if not callable(orientation): + orientation = get_orientation(orientation) + + return cutout._convert_cutout( + convert_func=convert_pv, + panel=panel, + orientation=orientation, + clearsky_model=clearsky_model, + **params, + ) + + +def pm25(cutout: Cutout, **params): + """ + Generate PM2.5 time series [ug / m3] + (see convert_pm25 for details) + + Returns: + xr.DataArray: PM2.5 time series + + """ + + return cutout._convert_cutout(convert_func=convert_pm25, **params) + + +__all__ = [ + "heat_demand", + "temperature", + "soil_temperature", + "solar_thermal", + "wind", + "windspd", + "windwpd", + "pv", + "pm25", +] diff --git a/src/geodata/cutout.py b/src/geodata/cutout.py index bae85889..507cf932 100644 --- a/src/geodata/cutout.py +++ b/src/geodata/cutout.py @@ -21,7 +21,6 @@ import logging from functools import partial -from operator import itemgetter from pathlib import Path from typing import Literal, Optional, Union @@ -34,14 +33,15 @@ from . import config from .convert import ( - convert_heat_demand, - convert_pm25, - convert_pv, - convert_solar_thermal, - convert_wind, - convert_windspd, - convert_windwpd, - get_orientation, + heat_demand, + pm25, + pv, + soil_temperature, + solar_thermal, + temperature, + wind, + windspd, + windwpd, ) from .datasets._base import BaseDataset from .mask import Mask @@ -51,7 +51,6 @@ cutout_prepare, cutout_produce_specific_dataseries, ) -from .resource import get_solarpanelconfig, get_windturbineconfig, windturbine_smooth from .types import BoundRange, DateRange from .utils import ensure_slice @@ -196,13 +195,14 @@ def _get_filename( def catalog(self): """A generator that yields all dataset files.""" - for year in self.coords["year"]: - for month in self.coords["month"]: + for year in self.coords["year"].values: + for month in self.coords["month"].values: yield self._get_filename(year, month) @property def meta_path(self): """Path to the metadata file.""" + return self._get_filename() @property @@ -506,316 +506,15 @@ def _convert_cutout( return xr.concat(results, dim="time") - def heat_demand( - self, - threshold: float = 15.0, - a: float = 1.0, - constant: float = 0.0, - hour_shift: float = 0.0, - **params, - ): - """Convert outside temperature into daily heat demand using the - degree-day approximation. - - Since "daily average temperature" means different things in - different time zones and since xarray coordinates do not handle - time zones gracefully like pd.DateTimeIndex, you can provide an - hour_shift to redefine when the day starts. - - E.g. for Moscow in winter, hour_shift = 4, for New York in winter, - hour_shift = -5 - - This time shift applies across the entire spatial scope of ds for - all times. More fine-grained control will be built in a some - point, i.e. space- and time-dependent time zones. - - WARNING: Because the original data is provided every month, at the - month boundaries there is untidiness if you use a time shift. The - resulting xarray will have duplicates in the index for the parts - of the day in each month at the boundary. You will have to - re-average these based on the number of hours in each month for - the duplicated day. - - Args: - threshold (float): Outside temperature in degrees Celsius above which there is no heat demand. - a (float): Linear factor relating heat demand to outside temperature. - constant (float): Constant part of heat demand that does not depend on outside - temperature (e.g. due to water heating). - hour_shift (float): Time shift relative to UTC for taking daily average - - Returns: - xr.DataArray: Heat demand - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - """ - - return self._convert_cutout( - convert_func=convert_heat_demand, - threshold=threshold, - a=a, - constant=constant, - hour_shift=hour_shift, - **params, - ) - - def temperature(self, **convert_params): - """Convert temperature in Cutout to outside temperature. - - Args: - convert_params: Keyword arguments passed to `convert_cutout` function - - Returns: - xr.DataArray: Data of the Cutout with temperature converted to outside temperatures. - """ - return self._convert_cutout( - convert_func=lambda ds: ds["temperature"] - 273.15, **convert_params - ) - - def soil_temperature(self, **convert_params): - """Return soil temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - - Args: - convert_params: Keyword arguments passed to `convert_cutout` function - - Returns: - xr.DataArray: Data of the Cutout with temperature converted to soil temperatures. - """ - return self._convert_cutout( - convert_func=lambda ds: (ds["soil temperature"] - 273.15).fillna(0.0), - **convert_params, - ) - - def solar_thermal( - self, - orientation: Optional[Union[dict, str, callable]] = None, - trigon_model: str = "simple", - clearsky_model: Literal["simple", "enhanced"] = "simple", - c0: float = 0.8, - c1: float = 3.0, - t_store: float = 80.0, - **params, - ): - """Convert downward short-wave radiation flux and outside temperature - into time series for solar thermal collectors. - - Mathematical model and defaults for c0, c1 based on model in [1]. - - Args: - orientation (Union[dict, str, callable]): Panel orientation with slope and azimuth - (units of degrees), or 'latitude_optimal'. - trigon_model (str): Type of trigonometry model - clearsky_model (str): Type of clearsky model for diffuse irradiation. Either - `simple` or `enhanced`. - c0 (float): Parameter for model in [1] This defaults to 0.8. - c1 (float): Parameter for model in [1] This defaults to 3.0. - t_store (float): Store temperature in degree Celsius - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Henning and Palzer, Renewable and Sustainable Energy Reviews 30 - (2014) 1003-1018 - """ - - if orientation is None: - orientation = {"slope": 45.0, "azimuth": 180.0} - - if not callable(orientation): - orientation = get_orientation(orientation) - - return self._convert_cutout( - convert_func=convert_solar_thermal, - orientation=orientation, - trigon_model=trigon_model, - clearsky_model=clearsky_model, - c0=c0, - c1=c1, - t_store=t_store, - **params, - ) - - def wind( - self, - turbine: Union[str, dict], - method: Literal["simple", "interpolation", "extrapolation"], - smooth: Union[bool, dict] = False, - **params, - ): - """Convert wind speed time-series into wind generation time-series. - - Args: - turbine (Union[str, dict]): Name of a turbine or a dictionary with the parameters - for the wind turbine in [2]. - smooth (Union[bool, dict]): If True, the wind speed time-series will be smoothed - before conversion. If False, no smoothing will be applied. If a dictionary is - passed, the smoothing parameters will be used. - **params: Keyword arguments passed to `convert_cutout` function - """ - - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - - if smooth: - turbine = windturbine_smooth(turbine, params=smooth) - - match method: - case "simple": - return self._convert_cutout( - convert_func=convert_wind, turbine=turbine, **params - ) - - case _: - raise ValueError(f"Method {method} not supported.") - - def windspd(self, **params): - """ - Generate wind speed time-series - - convert.convert_cutout → convert.convert_windspd - - Parameters - ---------- - **params - Must have 1 of: - turbine : str or dict - Name of a turbine - hub_height : num - Extrapolation height - - Can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - e.g. var_height='lml' - - """ - - if "turbine" in params: - turbine = params.pop("turbine") - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - else: - raise ValueError(f"Turbine ({turbine}) not found.") - hub_height = itemgetter("hub_height")(turbine) - elif "hub_height" in params: - hub_height = params.pop("hub_height") - elif "to_height" in params: - hub_height = params.pop("to_height") - else: - raise ValueError("Either a turbine or hub_height must be specified.") - - params["hub_height"] = hub_height - - return self._convert_cutout(convert_func=convert_windspd, **params) - - def windwpd(self, **params): - """ - Generate wind power density time-series - - convert.convert_cutout → convert.convert_windwpd - - Parameters - ---------- - **params - Must have 1 of: - turbine : str or dict - Name of a turbine - hub_height : num - Extrapolation height - - Can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - e.g. var_height='lml' - - """ - - if "turbine" in params: - turbine = params.pop("turbine") - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - else: - raise ValueError(f"Turbine ({turbine}) not found.") - hub_height = itemgetter("hub_height")(turbine) - elif "hub_height" in params: - hub_height = params.pop("hub_height") - elif "to_height" in params: - hub_height = params.pop("to_height") - else: - raise ValueError("Either a turbine or hub_height must be specified.") - - params["hub_height"] = hub_height - - return self._convert_cutout(convert_func=convert_windwpd, **params) - - def pv( - self, - panel: Union[str, dict], - orientation: Union[str, dict, callable], - clearsky_model: Optional[str] = None, - **params, - ): - """Convert downward-shortwave, upward-shortwave radiation flux and - ambient temperature into a pv generation time-series. - - Args: - panel (Union[str, dict]): Panel name known to the reatlas client or a panel config - dictionary with the parameters for the electrical model in [3]. - orientation (Union[str, dict, callback]): Panel orientation can be chosen from either - 'latitude_optimal', a constant orientation {'slope': 0.0, - 'azimuth': 0.0} or a callback function with the same signature - as the callbacks generated by the - `geodata.pv.orientation.make_*` functions. - clearsky_model (Optional[str]): Either the 'simple' or the 'enhanced' Reindl clearsky - model. The default choice of None will choose dependending on - data availability, since the 'enhanced' model also - incorporates ambient air temperature and relative humidity. - - Returns: - xr.DataArray: Time-series or capacity factors based on additional general - conversion arguments. - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Soteris A. Kalogirou. Solar Energy Engineering: Processes and Systems, - pages 49-117,469-516. Academic Press, 2009. ISBN 0123745012. - [2] D.T. Reindl, W.A. Beckman, and J.A. Duffie. Diffuse fraction correla- - tions. Solar Energy, 45(1):1 - 7, 1990. - [3] Hans Georg Beyer, Gerd Heilscher and Stefan Bofinger. A Robust Model - for the MPP Performance of Different Types of PV-Modules Applied for - the Performance Check of Grid Connected Systems, Freiburg, June 2004. - Eurosun (ISES Europe Solar Congress). - """ - - if isinstance(panel, str): - panel = get_solarpanelconfig(panel) - if not callable(orientation): - orientation = get_orientation(orientation) - - return self._convert_cutout( - convert_func=convert_pv, - panel=panel, - orientation=orientation, - clearsky_model=clearsky_model, - **params, - ) - - def pm25(self, **params): - """ - Generate PM2.5 time series [ug / m3] - (see convert_pm25 for details) - - Returns: - xr.DataArray: PM2.5 time series - - """ - - return self._convert_cutout(convert_func=convert_pm25, **params) + heat_demand = heat_demand + temperature = temperature + soil_temperature = soil_temperature + solar_thermal = solar_thermal + wind = wind + windspd = windspd + windwpd = windwpd + pm25 = pm25 + pv = pv def ds_reformat_index(ds: xr.DataArray) -> xr.DataArray: @@ -957,3 +656,6 @@ def calc_shp_area(shp, shp_projection="+proj=latlon"): shp, ) return temp_shape.area / 1000000 + + +__all__ = ["Cutout", "coarsen", "calc_grid_area", "calc_shp_area"] diff --git a/src/geodata/preparation.py b/src/geodata/preparation.py index d6fa0240..8370fb06 100644 --- a/src/geodata/preparation.py +++ b/src/geodata/preparation.py @@ -154,7 +154,7 @@ def datasetfn_with_id(ym): pool.map(cutout_do_task, tasks) except Exception as e: pool.terminate() - logger.info( + logger.warning( "Preparation of cutout '%s' has been interrupted by an exception. " "Purging the incomplete cutout_dir.", cutout.name, From 0a338c0620396d931d882b09169aa41070adb338 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 4 Mar 2025 05:19:24 +0000 Subject: [PATCH 21/54] fix: limit wildcard import from plot module --- src/geodata/plot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/geodata/plot.py b/src/geodata/plot.py index 11e16f04..f6e47872 100644 --- a/src/geodata/plot.py +++ b/src/geodata/plot.py @@ -479,3 +479,6 @@ def save_animation(file_name: str): """ ) return javascript + + +__all__ = ["time_series", "heatmap", "heatmap_animation", "save_animation"] From d212392c9bee943223d8ea8e5a865a8a59011309 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 4 Mar 2025 05:20:05 +0000 Subject: [PATCH 22/54] draft: ERA5 Wind 3D Dataset --- src/geodata/datasets/era5/_base.py | 41 +++++++- src/geodata/datasets/era5/hourly/__init__.py | 3 +- src/geodata/datasets/era5/hourly/wind_3d.py | 103 +++++++++++++++++++ src/geodata/types.py | 8 ++ 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 src/geodata/datasets/era5/hourly/wind_3d.py diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index 7b5824f8..a6a8c330 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -14,16 +14,25 @@ # along with this program. If not, see . import logging +import os -import cdsapi import numpy as np import xarray as xr -from ...types import CoordRange +from ...types import CoordRange, PathLike from .._base import BaseDataset logger = logging.getLogger(__name__) +try: + import cdsapi +except ImportError: + logger.warning( + "cdsapi is not installed. You will not be able to download ERA5 data." + "Please install it with `pip install 'geodata-re[download]'." + ) + cdsapi = None + def _convert_and_subset_lons_lats_era5(ds: xr.Dataset, xs: slice, ys: slice): # Rename geographic dimensions to x,y @@ -124,3 +133,31 @@ def tasks_func( ) for year, month in yearmonths ] + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month.""" + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn) as ds: + logger.info("Opening %s", fn) + ds = _subset_x_y_era5(ds, xs, ys) + + # New ERA5 format for hourly datasets + # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 + # TODO: We can remove this if we refactor geodata's convert module in the future + if "valid_time" in ds.coords: + ds = ds.rename({"valid_time": "time"}) + + yield (year, month), ds diff --git a/src/geodata/datasets/era5/hourly/__init__.py b/src/geodata/datasets/era5/hourly/__init__.py index fbaa1656..b76666fc 100644 --- a/src/geodata/datasets/era5/hourly/__init__.py +++ b/src/geodata/datasets/era5/hourly/__init__.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from .wind_3d import ERA5Wind3DHourlyDataset from .wind_solar import ERA5WindSolarHourlyDataset -__all__ = ["ERA5WindSolarHourlyDataset"] +__all__ = ["ERA5WindSolarHourlyDataset", "ERA5Wind3DHourlyDataset"] diff --git a/src/geodata/datasets/era5/hourly/wind_3d.py b/src/geodata/datasets/era5/hourly/wind_3d.py new file mode 100644 index 00000000..9c3d6645 --- /dev/null +++ b/src/geodata/datasets/era5/hourly/wind_3d.py @@ -0,0 +1,103 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import pprint +import tempfile +from calendar import monthrange +from pathlib import Path + +import xarray as xr + +from ..._base import AtomicDataset +from .._base import ERA5BaseDataset + +logger = logging.getLogger(__name__) + + +class ERA5Wind3DHourlyDataset(ERA5BaseDataset): + """ERA5Wind3DHourlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind information. + This dataset is stored in hourly intervals. + """ + + weather_config = "wind_3d_hourly" + L137_LEVELS = range(131, 138) + + # Information that are needed for ERA5's API request + variables = {"u": "131", "v": "132"} + product = "reanalysis-era5-complete" + + def _download_file(self, file: AtomicDataset): + """Sample request for reference: + + c.retrieve('reanalysis-era5-complete', { + 'date' : '20130101/to/20130131', + 'levelist': '1/10/100/137', + 'levtype' : 'ml', + 'param' : '130, # Full information at https://apps.ecmwf.int/codes/grib/param-db/ + 'stream' : 'oper', # Denotes ERA5. Ensemble members are selected by 'enda' + 'time' : '00/to/23/by/6', + 'type' : 'an', + 'grid' : '1.0/1.0', + 'format' : 'netcdf', + }, 'save_path.nc') # Output file. Adapt as you wish. + """ + + year: int = file.year + month: int = file.month + save_path: Path = file.path + + if self.testing: + date = f"{year}{month:02d}01/to/{year}{month:02d}03" + else: + date = ( + f"{year}{month:02d}01/to/{year}{month:02d}{monthrange(year, month)[1]}" + ) + full_request = { + "date": date, + "levelist": "/".join([str(level) for level in self.L137_LEVELS]), + "levtype": "ml", + "param": "/".join(self.variables.values()), + "stream": "oper", + "time": "00/to/23/by/1", + "type": "an", + "grid": "0.25/0.25", # NOTE: We want the highest resolution possible + "format": "netcdf", + } + + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + + full_result = self.client.retrieve(self.product, full_request) + + if not self.bounds: + full_result.download(save_path) + logger.info("File downloaded: %s", save_path) + return + + # NOTE: Raw MARS request doesn't support bounding box, so we need to + # subset the data after downloading + with tempfile.NamedTemporaryFile(suffix=".nc") as tmpfile: + full_result.download(tmpfile.name) + with xr.open_dataset(tmpfile.name, chunks="auto") as ds: + ds = ds.sel( + longitude=slice(*sorted([self.bounds[0], self.bounds[2]])), + latitude=slice( + *sorted([self.bounds[1], self.bounds[3]], reverse=True) + ), + ) + ds.to_netcdf(save_path) + + logger.info("File downloaded: %s", save_path) diff --git a/src/geodata/types.py b/src/geodata/types.py index 70dafba0..9bf5715f 100644 --- a/src/geodata/types.py +++ b/src/geodata/types.py @@ -13,8 +13,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +"""This module defines some of the common types used in the geodata package.""" + +from pathlib import Path + +# Geodata Specific Types DateRange = slice | tuple[int, int] | list[int] CoordRange = slice | tuple[float, float] | list[float] BoundRange = tuple[float, float, float, float] | list[float] +# Filesystem +PathLike = str | Path + __all__ = ["DateRange", "CoordRange", "BoundRange"] From f23847fd00b4f64f3c7cdeb4d314ba99b55bf2ab Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 28 Mar 2025 17:47:05 -0700 Subject: [PATCH 23/54] feat: transition ERA5 3D wind to daily storage due to CDS OOM issue --- src/geodata/datasets/_base.py | 4 +++- src/geodata/datasets/era5/hourly/wind_3d.py | 11 +++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index b861f57c..4b140c63 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -260,7 +260,9 @@ def _download_file(self, file: AtomicDataset): def download(self, force: bool = False): """Method to download the dataset. This method should download the - dataset files and store them in the appropriate location. + dataset files and store them in the appropriate location. If the dataset + is specified as a ``testing'' dataset, only the first three (3) days + of the dataset will be downloaded. Args: force: A boolean flag indicating whether to force the download of diff --git a/src/geodata/datasets/era5/hourly/wind_3d.py b/src/geodata/datasets/era5/hourly/wind_3d.py index 9c3d6645..1bd0ef3d 100644 --- a/src/geodata/datasets/era5/hourly/wind_3d.py +++ b/src/geodata/datasets/era5/hourly/wind_3d.py @@ -16,7 +16,6 @@ import logging import pprint import tempfile -from calendar import monthrange from pathlib import Path import xarray as xr @@ -34,6 +33,7 @@ class ERA5Wind3DHourlyDataset(ERA5BaseDataset): """ weather_config = "wind_3d_hourly" + frequency = "daily" L137_LEVELS = range(131, 138) # Information that are needed for ERA5's API request @@ -58,16 +58,11 @@ def _download_file(self, file: AtomicDataset): year: int = file.year month: int = file.month + day: int = file.day save_path: Path = file.path - if self.testing: - date = f"{year}{month:02d}01/to/{year}{month:02d}03" - else: - date = ( - f"{year}{month:02d}01/to/{year}{month:02d}{monthrange(year, month)[1]}" - ) full_request = { - "date": date, + "date": f"{year}{month:02d}{day:02d}", "levelist": "/".join([str(level) for level in self.L137_LEVELS]), "levtype": "ml", "param": "/".join(self.variables.values()), From 3e2ff8a2e90c00a4dc1abc9c4f29767e7417c338 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sun, 30 Mar 2025 14:21:46 -0700 Subject: [PATCH 24/54] fix: skip atomic files during download if they are already present --- src/geodata/datasets/_base.py | 5 +++++ src/geodata/datasets/era5/_base.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 4b140c63..49d54316 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -274,6 +274,11 @@ def download(self, force: bool = False): return for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): + # Skip the file if it has already been downloaded (unless force is True) + if file.check() and not force: + logger.debug(f"{file.path} already exists, skipping download") + continue + # We first must ensure the directory exists file.path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index a6a8c330..ecc1ef11 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -35,9 +35,6 @@ def _convert_and_subset_lons_lats_era5(ds: xr.Dataset, xs: slice, ys: slice): - # Rename geographic dimensions to x,y - # Subset x,y according to xs, ys (subset_x_y_era5) - # Longitudes should go from -180. to +180. if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): ds = xr.concat( @@ -161,3 +158,6 @@ def prepare_func( ds = ds.rename({"valid_time": "time"}) yield (year, month), ds + + def _dataset_postprocess(self, ds, **kwargs): + return super()._dataset_postprocess(ds, **kwargs) From 8681039531f31b85d5968eeeb9ab7b488b206db9 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:17:00 -0700 Subject: [PATCH 25/54] feat: MERRA2 support for multi-file datasets with slv_flux_hourly --- src/geodata/datasets/_base.py | 13 +- src/geodata/datasets/merra2/_base.py | 112 ++++++++++++++++-- .../datasets/merra2/hourly/__init__.py | 3 +- .../datasets/merra2/hourly/slv_flux.py | 52 ++++++++ .../datasets/merra2/hourly/surface_flux.py | 4 +- 5 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 src/geodata/datasets/merra2/hourly/slv_flux.py diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 49d54316..f36661e9 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -44,7 +44,7 @@ class AtomicDataset: month: int day: int | None = None file_hash: str | None = None - url: str | None = None + url: str | tuple[str] | None = None spinup: bool | None = None def __post_init__(self): @@ -284,13 +284,9 @@ def download(self, force: bool = False): self._download_file(file) - logger.info(f"Downloaded {self}") - logger.info("Cleaning and renaming coordinates") - - # Post-process the dataset - for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): if file.check(): - ds = xr.open_dataset(file.path).chunk() + logger.debug("Postprocessing %s", file.path) + ds = xr.open_dataset(file.path).chunk("auto") ds = self._rename_and_clean_coords(ds) ds = self._dataset_postprocess(ds) @@ -302,6 +298,9 @@ def download(self, force: bool = False): file.path.unlink() file.path.with_stem(file.path.stem + "_postprocessed").rename(file.path) + logger.info(f"Downloaded {self}") + logger.info("Cleaning and renaming coordinates") + def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): """Method to postprocess the dataset after it has been downloaded. This method should be implemented by subclasses to handle any diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index d09a1050..2571c465 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -14,13 +14,17 @@ # along with this program. If not, see . import logging +import os +import tempfile from calendar import monthrange +from typing import Sequence import numpy as np import requests import xarray as xr +from tqdm.auto import tqdm -from ...types import CoordRange +from ...types import CoordRange, PathLike from .._base import AtomicDataset, BaseDataset logger = logging.getLogger(__name__) @@ -60,28 +64,87 @@ class MERRA2BaseDataset(BaseDataset): """MERRA2BaseDataset is a class that encaps a dataset from the MERRA2 reanalysis dataset. It provides a streamlined workflow for downloading, preprocessing, and storing of these datasets. - - TODO: Support multi-file downloads. """ + url_template: Sequence[str] + variables: Sequence[str] module = "merra2" projection = "latlong" lat_direction = True frequency = "daily" - url_template = "" def _download_file(self, file: AtomicDataset): - assert "url" in file, "URL is required to download the file" + assert hasattr(file, "url"), "URL is required to download the file" - url: str = file.url + url: tuple[str] = file.url # Download the file - with requests.get(url, stream=True) as r: - r.raise_for_status() + match len(file.url): + case 1: + logger.debug("Downloading %s", url[0]) + with requests.get(url[0], stream=True) as r: + r.raise_for_status() + + with ( + open(file.path, "wb") as f, + tqdm( + total=int(r.headers.get("content-length", 0)), + unit="B", + unit_scale=True, + ) as pbar, + ): + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + pbar.update(len(chunk)) + case _: + with tempfile.TemporaryDirectory() as tempdir: + for i, url in enumerate(file.url): + logger.debug("Downloading %s", url) + with requests.get(url, stream=True) as r: + r.raise_for_status() + + with ( + open(os.path.join(tempdir, str(i)), "wb") as f, + tqdm( + total=int(r.headers.get("content-length", 0)), + unit="B", + unit_scale=True, + ) as pbar, + ): + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + pbar.update(len(chunk)) + + with xr.open_mfdataset( + [os.path.join(tempdir, str(i)) for i in range(len(file.url))], + combine="by_coords", + chunks="auto", + ) as ds: + ds.to_netcdf(file.path) + logger.info("Preprocessing complete with multiple files") + + def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): + """Postprocess the dataset after downloading and opening it. + + Args: + ds (xr.Dataset): The dataset to postprocess. + **kwargs: Additional keyword arguments. + + Returns: + xr.Dataset: The postprocessed dataset. + """ + + # Rename the coordinates + ds = ds.rename({"x": "lon", "y": "lat"}) + ds = ds.assign_coords(lon=ds.coords["lon"], lat=ds.coords["lat"]) + + # Only keep the variables that are needed + for var in ds.data_vars: + if var.lower() not in self.variables: + ds = ds.drop(var) - with open(file.path, "wb") as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) + # Change all variables to lowercase + return ds.rename({var: var.lower() for var in ds.data_vars}) def spinup_year(self, year: int, month: int): """Returns the spinup period for the given year and month. @@ -118,7 +181,7 @@ def _daily_catalog(self): for file in catalog: file.spinup = self.spinup_year(file.year, file.month) - file.url = self.url_template.format(**vars(file)) + file.url = [t.format(**vars(file)) for t in self.url_template] return catalog @@ -190,3 +253,28 @@ def tasks_func( ] case _: raise NotImplementedError("Frequency not supported") + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month.""" + + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn) as ds: + logger.info("Opening %s", fn) + ds = _convert_and_subset_lons_lats_merra2(ds, xs, ys) + ds = ds.rename({"x": "lon", "y": "lat"}) + ds = ds.assign_coords(lon=ds.coords["lon"], lat=ds.coords["lat"]) + + yield (year, month), ds diff --git a/src/geodata/datasets/merra2/hourly/__init__.py b/src/geodata/datasets/merra2/hourly/__init__.py index cd3c4962..5f96ae88 100644 --- a/src/geodata/datasets/merra2/hourly/__init__.py +++ b/src/geodata/datasets/merra2/hourly/__init__.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from .slv_flux import MERRA2SLVFluxHourlyDataset from .surface_flux import MERRA2SurfaceFluxHourlyDataset -__all__ = ["MERRA2SurfaceFluxHourlyDataset"] +__all__ = ["MERRA2SurfaceFluxHourlyDataset", "MERRA2SLVFluxHourlyDataset"] diff --git a/src/geodata/datasets/merra2/hourly/slv_flux.py b/src/geodata/datasets/merra2/hourly/slv_flux.py new file mode 100644 index 00000000..3edfb8c6 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/slv_flux.py @@ -0,0 +1,52 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVFluxHourlyDataset(MERRA2BaseDataset): + """MERRA2SLVFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_flux_hourly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + "u2m", + "v2m", + "u10m", + "v10m", + "u50m", + "v50m", + ] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/surface_flux.py b/src/geodata/datasets/merra2/hourly/surface_flux.py index 5f8a4af4..523b470b 100644 --- a/src/geodata/datasets/merra2/hourly/surface_flux.py +++ b/src/geodata/datasets/merra2/hourly/surface_flux.py @@ -39,4 +39,6 @@ class MERRA2SurfaceFluxHourlyDataset(MERRA2BaseDataset): "hflux", "eflux", ] - url_template = "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4" + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) From 6a5711dfaccf137ac321decc9bc206c97bc6dffd Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:35:38 -0700 Subject: [PATCH 26/54] feat: add MERRA2 daily and hourly datasets with surface flux, radiation, and aerosol support --- src/geodata/datasets/__init__.py | 3 +- src/geodata/datasets/_base.py | 21 +++++++++++- src/geodata/datasets/merra2/__init__.py | 4 +-- src/geodata/datasets/merra2/daily/__init__.py | 18 ++++++++++ .../datasets/merra2/daily/surface_flux.py | 31 +++++++++++++++++ .../datasets/merra2/hourly/__init__.py | 9 ++++- .../datasets/merra2/hourly/slv_radiation.py | 33 +++++++++++++++++++ .../datasets/merra2/hourly/surface_aerosol.py | 32 ++++++++++++++++++ 8 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 src/geodata/datasets/merra2/daily/__init__.py create mode 100644 src/geodata/datasets/merra2/daily/surface_flux.py create mode 100644 src/geodata/datasets/merra2/hourly/slv_radiation.py create mode 100644 src/geodata/datasets/merra2/hourly/surface_aerosol.py diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index f874c8ad..21e33c7c 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -15,5 +15,6 @@ # along with this program. If not, see . from . import era5, merra2 +from ._base import _registry as registry -__all__ = ["era5", "merra2"] +__all__ = ["era5", "merra2", "registry"] diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index f36661e9..24485bec 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -20,7 +20,7 @@ import logging from collections.abc import Sequence from pathlib import Path -from typing import Literal +from typing import Literal, Type import pandas as pd import xarray as xr @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) +_registry: dict[str, Type["BaseDataset"]] = {} + @dataclasses.dataclass class AtomicDataset: @@ -518,3 +520,20 @@ def _get_files(cls, year: int, month: int): raise ValueError( f"Invalid frequency {cls.frequency} defined for this dataset." ) + + def __init_subclass__(cls: Type["BaseDataset"], **kwargs): + """Register the subclass in the registry. This allows us to + dynamically load the dataset from the module. + + Args: + cls: The class of the dataset. + """ + + super().__init_subclass__(**kwargs) + if not hasattr(cls, "weather_config"): + # Skip the class if it does not have a weather_config attribute + # This could be the case with intermediate base classes + return + + _registry[cls.weather_config] = cls + logger.debug(f"Registered {cls.weather_config} in the registry") diff --git a/src/geodata/datasets/merra2/__init__.py b/src/geodata/datasets/merra2/__init__.py index 334f2004..57776272 100644 --- a/src/geodata/datasets/merra2/__init__.py +++ b/src/geodata/datasets/merra2/__init__.py @@ -13,6 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from . import hourly +from . import daily, hourly -__all__ = ["hourly"] +__all__ = ["hourly", "daily"] diff --git a/src/geodata/datasets/merra2/daily/__init__.py b/src/geodata/datasets/merra2/daily/__init__.py new file mode 100644 index 00000000..7bf31a96 --- /dev/null +++ b/src/geodata/datasets/merra2/daily/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .surface_flux import MERRA2SurfaceFluxDailyDataset + +__all__ = ["MERRA2SurfaceFluxDailyDataset"] diff --git a/src/geodata/datasets/merra2/daily/surface_flux.py b/src/geodata/datasets/merra2/daily/surface_flux.py new file mode 100644 index 00000000..4cd4f577 --- /dev/null +++ b/src/geodata/datasets/merra2/daily/surface_flux.py @@ -0,0 +1,31 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxDailyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_daily" + + variables = ["hournorain", "tprecmax", "t2mmax", "t2mmean", "t2mmin"] + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2SDNXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/__init__.py b/src/geodata/datasets/merra2/hourly/__init__.py index 5f96ae88..29db1c06 100644 --- a/src/geodata/datasets/merra2/hourly/__init__.py +++ b/src/geodata/datasets/merra2/hourly/__init__.py @@ -14,6 +14,13 @@ # along with this program. If not, see . from .slv_flux import MERRA2SLVFluxHourlyDataset +from .slv_radiation import MERRA2SLVRadiationHourlyDataset +from .surface_aerosol import MERRA2SurfaceAerosolHourlyDataset from .surface_flux import MERRA2SurfaceFluxHourlyDataset -__all__ = ["MERRA2SurfaceFluxHourlyDataset", "MERRA2SLVFluxHourlyDataset"] +__all__ = [ + "MERRA2SurfaceFluxHourlyDataset", + "MERRA2SLVFluxHourlyDataset", + "MERRA2SLVRadiationHourlyDataset", + "MERRA2SurfaceAerosolHourlyDataset", +] diff --git a/src/geodata/datasets/merra2/hourly/slv_radiation.py b/src/geodata/datasets/merra2/hourly/slv_radiation.py new file mode 100644 index 00000000..74c61739 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/slv_radiation.py @@ -0,0 +1,33 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVRadiationHourlyDataset(MERRA2BaseDataset): + """MERRA2SLVRadiationHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_radiation_hourly" + + variables = ["albedo", "swgdn", "swtdn", "t2m"] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/surface_aerosol.py b/src/geodata/datasets/merra2/hourly/surface_aerosol.py new file mode 100644 index 00000000..c1ade9f0 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/surface_aerosol.py @@ -0,0 +1,32 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceAerosolHourlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceAerosolHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_aerosol_hourly" + + variables = ["bcsmass", "dusmass25", "ocsmass", "so4smass", "sssmass25"] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXAER.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) From fa53f564df07b4e64b0cba2be1dd9dd517b07266 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:43:20 -0700 Subject: [PATCH 27/54] feat: add monthly datasets for MERRA2 including surface flux and radiation support --- src/geodata/datasets/_base.py | 4 +- .../datasets/era5/monthly/wind_solar.py | 1 + src/geodata/datasets/merra2/__init__.py | 4 +- src/geodata/datasets/merra2/_base.py | 12 +++++ .../datasets/merra2/monthly/__init__.py | 19 ++++++++ .../datasets/merra2/monthly/slv_radiation.py | 33 ++++++++++++++ .../datasets/merra2/monthly/surface_flux.py | 45 +++++++++++++++++++ 7 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 src/geodata/datasets/merra2/monthly/__init__.py create mode 100644 src/geodata/datasets/merra2/monthly/slv_radiation.py create mode 100644 src/geodata/datasets/merra2/monthly/surface_flux.py diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 24485bec..0529437b 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -429,8 +429,6 @@ def catalog(self) -> list["AtomicDataset"]: cat = self._monthly_catalog() case "daily": cat = self._daily_catalog() - case "hourly": - cat = self._hourly_catalog() case _: raise ValueError( f"Invalid frequency {self.frequency} defined for this dataset." @@ -438,7 +436,7 @@ def catalog(self) -> list["AtomicDataset"]: return cat - def _monthly_catalog(self): + def _monthly_catalog(self) -> list["AtomicDataset"]: catalog = [] for year, month in itertools.product( diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py index 9c1d4172..bebfe36b 100644 --- a/src/geodata/datasets/era5/monthly/wind_solar.py +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -55,6 +55,7 @@ class ERA5WindSolarMonthlyDataset(ERA5WindSolarHourlyDataset): """ weather_config = "wind_solar_monthly" + frequency = "monthly" def _download_file(self, file: AtomicDataset): year: int = file.year diff --git a/src/geodata/datasets/merra2/__init__.py b/src/geodata/datasets/merra2/__init__.py index 57776272..a3c064f4 100644 --- a/src/geodata/datasets/merra2/__init__.py +++ b/src/geodata/datasets/merra2/__init__.py @@ -13,6 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from . import daily, hourly +from . import daily, hourly, monthly -__all__ = ["hourly", "daily"] +__all__ = ["hourly", "daily", "monthly"] diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 2571c465..194bc47d 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -185,6 +185,18 @@ def _daily_catalog(self): return catalog + def _monthly_catalog(self): + if not self.url_template: + raise NotImplementedError("url_template is not defined for this dataset") + + catalog = super()._monthly_catalog() + + for file in catalog: + file.spinup = self.spinup_year(file.year, file.month) + file.url = [t.format(**vars(file)) for t in self.url_template] + + return catalog + @classmethod def meta_prepare_func( cls, xs: CoordRange, ys: CoordRange, year: int, month: int, **params diff --git a/src/geodata/datasets/merra2/monthly/__init__.py b/src/geodata/datasets/merra2/monthly/__init__.py new file mode 100644 index 00000000..4fc2414b --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .slv_radiation import MERRA2SLVRadiationMonthlyDataset +from .surface_flux import MERRA2SurfaceFluxMonthlyDataset + +__all__ = ["MERRA2SLVRadiationMonthlyDataset", "MERRA2SurfaceFluxMonthlyDataset"] diff --git a/src/geodata/datasets/merra2/monthly/slv_radiation.py b/src/geodata/datasets/merra2/monthly/slv_radiation.py new file mode 100644 index 00000000..201d1bdb --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/slv_radiation.py @@ -0,0 +1,33 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVRadiationMonthlyDataset(MERRA2BaseDataset): + """MERRA2SLVRadiationMonthlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_radiation_monthly" + frequency = "monthly" + variables = ["albedo", "swgdn", "swtdn", "t2m"] + + url_template = [ + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXSLV.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_slv_Nx.{year}{month:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXRAD.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_rad_Nx.{year}{month:0>2}.nc4", + ] diff --git a/src/geodata/datasets/merra2/monthly/surface_flux.py b/src/geodata/datasets/merra2/monthly/surface_flux.py new file mode 100644 index 00000000..f3f71300 --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/surface_flux.py @@ -0,0 +1,45 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxMonthlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxMonthlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_monthly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + ] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXFLX.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4", + ) From 6ac68a3e3307558b68a8cb273b8e986e6353f284 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 10 Apr 2025 14:56:14 -0700 Subject: [PATCH 28/54] faet: support automatic retry for ERA datasets --- src/geodata/datasets/era5/hourly/wind_3d.py | 26 ++++++++++++++++--- .../datasets/era5/hourly/wind_solar.py | 12 ++++++++- .../datasets/era5/monthly/wind_solar.py | 12 ++++++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/geodata/datasets/era5/hourly/wind_3d.py b/src/geodata/datasets/era5/hourly/wind_3d.py index 1bd0ef3d..95aeef4f 100644 --- a/src/geodata/datasets/era5/hourly/wind_3d.py +++ b/src/geodata/datasets/era5/hourly/wind_3d.py @@ -78,14 +78,32 @@ def _download_file(self, file: AtomicDataset): full_result = self.client.retrieve(self.product, full_request) if not self.bounds: - full_result.download(save_path) - logger.info("File downloaded: %s", save_path) - return + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(save_path) + logger.info("File downloaded: %s", save_path) + return + except Exception as e: + logger.error("Download failed: %s", e) + if _count == 3: + raise # NOTE: Raw MARS request doesn't support bounding box, so we need to # subset the data after downloading with tempfile.NamedTemporaryFile(suffix=".nc") as tmpfile: - full_result.download(tmpfile.name) + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(tmpfile.name) + logger.info("File downloaded: %s", save_path) + break + except Exception as e: + logger.error("Download failed: %s", e) + if _count == 3: + raise with xr.open_dataset(tmpfile.name, chunks="auto") as ds: ds = ds.sel( longitude=slice(*sorted([self.bounds[0], self.bounds[2]])), diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py index 377f6b7e..d376f93f 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -101,7 +101,17 @@ def _download_file(self, file: AtomicDataset): ) with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(os.path.join(tempdir, "download.zip")) + break + except Exception as e: + logger.error("Error downloading file: %s", e) + if _count == 3: + raise + with zipfile.ZipFile( os.path.join(tempdir, "download.zip"), "r" ) as zip_ref: diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py index bebfe36b..bcb3ae5a 100644 --- a/src/geodata/datasets/era5/monthly/wind_solar.py +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -84,7 +84,17 @@ def _download_file(self, file: AtomicDataset): ) with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(os.path.join(tempdir, "download.zip")) + break + except Exception as e: + logger.error("Error downloading file: %s", e) + if _count == 3: + raise + with zipfile.ZipFile( os.path.join(tempdir, "download.zip"), "r" ) as zip_ref: From a86bdfa88b17122e0b3d43e9a52dc6129d613312 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:20:13 -0700 Subject: [PATCH 29/54] feat: register HRRR dataset and update weather configuration for hourly wind datasets --- src/geodata/datasets/__init__.py | 7 ++++++- src/geodata/datasets/hrrr/hourly/wind_3d.py | 2 +- src/geodata/datasets/hrrr/hourly/wind_solar.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index 21e33c7c..46017244 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -17,4 +17,9 @@ from . import era5, merra2 from ._base import _registry as registry -__all__ = ["era5", "merra2", "registry"] +__all__ = ["era5", "merra2", "registry", "register_hrrr"] + + +def register_hrrr(): + """Register the HRRR dataset with the registry.""" + from . import hrrr # noqa: F401 diff --git a/src/geodata/datasets/hrrr/hourly/wind_3d.py b/src/geodata/datasets/hrrr/hourly/wind_3d.py index 4310ac9c..a043e375 100644 --- a/src/geodata/datasets/hrrr/hourly/wind_3d.py +++ b/src/geodata/datasets/hrrr/hourly/wind_3d.py @@ -45,7 +45,7 @@ class HRRR3DWindHourlyDataset(HRRRBaseDataset): - x: longitude """ - weather_config = "wind_3d" + weather_config = "hrrr_wind_3d_hourly" product = "nat" # Use "nat" product for 3D data def _download_file(self, file: AtomicDataset): diff --git a/src/geodata/datasets/hrrr/hourly/wind_solar.py b/src/geodata/datasets/hrrr/hourly/wind_solar.py index 5252fc14..e08c5ce0 100644 --- a/src/geodata/datasets/hrrr/hourly/wind_solar.py +++ b/src/geodata/datasets/hrrr/hourly/wind_solar.py @@ -42,7 +42,7 @@ class HRRRHourlyDataset(HRRRBaseDataset): as well as the variables they wish to download. """ - weather_config = "wind_solar" + weather_config = "hrrr_wind_solar_hourly" product = "sfc" def _download_file(self, file: AtomicDataset): From e29fa95ed0a7ec95328c47a6cb5d7bee05d94097 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:23:34 -0700 Subject: [PATCH 30/54] feat: update HRRR dataset processing to reindex time and remove valid_time coordinate --- src/geodata/datasets/hrrr/_base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index 4726b680..1525c5d1 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -78,7 +78,10 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): # to include all hours in the range logger.debug("Reindexing dataset to include all hours in the range") - ds = ds.resample(time="1h").mean() + dt: np.datetime64 = ( + ds["time"].values[0].astype("datetime64[D]").astype("datetime64[m]") + ) + ds = ds.reindex(time=pd.date_range(dt, periods=24, freq="h")).ffill(dim="time") # NOTE: For some reasons, HRRR's longitude is in the range of [0, 360] # instead of [-180, 180]. We need to put it back to [-180, 180]. @@ -86,6 +89,7 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): ds["x"] = (ds["x"] % 360 + 540) % 360 - 180 ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + del ds["valid_time"] return ds From aa38c44ed5002db1dd8ac165ae2991cda5028c43 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:23:44 -0700 Subject: [PATCH 31/54] fix: update model to conform with new dataset --- src/geodata/model/_base.py | 172 +++++--------------------- src/geodata/model/wind/extrapolate.py | 38 ------ src/geodata/model/wind/interpolate.py | 39 ------ 3 files changed, 32 insertions(+), 217 deletions(-) diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 3174c7e3..374dac2a 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -19,15 +19,12 @@ import json import shutil from pathlib import Path -from typing import Optional, Union +from typing import Optional import xarray as xr -import geodata - from ..config import model_dir -from ..cutout import Cutout -from ..dataset import Dataset +from ..datasets._base import BaseDataset from ..logging import logger from ..utils import NpEncoder @@ -37,7 +34,7 @@ class BaseModel(abc.ABC): Args: name (str): The name of the model. - source (Union[geodata.Dataset, geodata.Cutout, xr.Dataset]): The source of the model. Can be a Dataset, Cutout or xarray.Dataset. + source (BaseDataset): The source of the model. interpolate (bool, optional): Interpolate the source to the same grid as the target. Defaults to False. **kwargs: Additional keyword arguments to pass to the model. """ @@ -54,56 +51,37 @@ class BaseModel(abc.ABC): "weather_data_config", } - def __init__(self, source: Union[geodata.Dataset, geodata.Cutout], **kwargs): - if source.config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: + def __init__(self, source: BaseDataset, **kwargs): + if not isinstance(source, BaseDataset): + raise ValueError(f"Source must be a Dataset, but got {type(source)}.") + if source.weather_config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: raise ValueError( - f"Weather data config {source.config} is not supported by this model." + f"Weather data config {source.weather_config} is not supported by this model." ) - if not source.prepared: - raise ValueError( - "The source Dataset/Cutout for this model is not prepared." - ) + if not source.downloaded: + raise ValueError("The source Dataset for this model is not prepared.") self.source = source self._extra_kwargs = kwargs self._corrupt_metadata = False self._prepared = False - if isinstance(source, geodata.Dataset): - self._ref_path = model_dir.parent / self.source.module - self._path = ( - model_dir - / self.type - / self.__class__.__name__ - / "datasets" - / self.source.module - ) - else: - self._ref_path = model_dir.parent / "cutouts" - self._path = ( - model_dir - / self.type - / self.__class__.__name__ - / "cutouts" - / self.source.name - ) + self._ref_path = model_dir.parent / self.source.module + self._path = ( + model_dir / self.type / self.__class__.__name__ / self.source.module + ) if (meta_path := self._path / "meta.json").exists(): try: with open(meta_path, encoding="utf_8") as f: self.metadata = json.load(f) - # NOTE: Double-check if we have an updated dataset/cutout + # NOTE: Double-check if we have an updated dataset # If we do, we need to re-prepare the model - if isinstance(source, geodata.Dataset): - _source_files = self.extract_dataset_metadata(source).get( - "files_orig", {} - ) - else: - _source_files = self.extract_cutout_metadata(source).get( - "files_orig", {} - ) + _source_files = self.extract_dataset_metadata(source).get( + "files_orig", {} + ) if set(_source_files.keys()) != set( self.metadata.get("files_orig", {}).keys() @@ -113,10 +91,7 @@ def __init__(self, source: Union[geodata.Dataset, geodata.Cutout], **kwargs): meta_path, ) self._corrupt_metadata = True - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) + self.metadata = self.extract_dataset_metadata(source) except json.JSONDecodeError: logger.warning( @@ -124,16 +99,11 @@ def __init__(self, source: Union[geodata.Dataset, geodata.Cutout], **kwargs): meta_path, ) self._corrupt_metadata = True - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) + self.metadata = self.extract_dataset_metadata(source) + else: # NOTE: We only store metadata in transient fashion until preparation is done - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) + self.metadata = self.extract_dataset_metadata(source) def __repr__(self): return f"Model(source={self.source}, type={self.type})" @@ -176,17 +146,7 @@ def estimate( xr.DataArray: Dataset with wind speed. """ - if self.from_dataset: - return self._estimate_dataset( - height=height, - years=years, - months=months, - xs=xs, - ys=ys, - use_real_data=use_real_data, - ) - - return self._estimate_cutout( + return self._estimate_dataset( height=height, years=years, months=months, @@ -195,8 +155,8 @@ def estimate( use_real_data=use_real_data, ) - def extract_dataset_metadata(self, dataset: Dataset) -> dict: - if not dataset.prepared: + def extract_dataset_metadata(self, dataset: BaseDataset) -> dict: + if not dataset.downloaded: raise ValueError("The source dataset for this model is not prepared.") logger.info("Using dataset %s", dataset.module) @@ -211,43 +171,16 @@ def extract_dataset_metadata(self, dataset: Dataset) -> dict: if isinstance(dataset.months, slice): metadata["months"] = dataset.months.start, dataset.months.stop - metadata["weather_data_config"] = dataset.config + metadata["weather_data_config"] = dataset.weather_config # NOTE: file paths for estimation parameters will be added later in the prepare step metadata["files_prepared"] = {} metadata["files_orig"] = {} - for c, fp in dataset.downloadedFiles: - if c == metadata["weather_data_config"]: - with open(fp, "rb") as f: - metadata["files_orig"][ - str(Path(fp).relative_to(self._ref_path)) - ] = hashlib.sha256(f.read()).hexdigest() - - return metadata - - def extract_cutout_metadata(self, cutout: Cutout) -> dict: - logger.info("Using cutout %s", cutout.name) - - metadata = {} - - metadata["name"] = cutout.name - metadata["module"] = cutout.meta.attrs["module"] - metadata["from_dataset"] = False - metadata["weather_data_config"] = cutout.config - - if isinstance(cutout.years, slice): - metadata["years"] = cutout.years.start, cutout.years.stop - if isinstance(cutout.months, slice): - metadata["months"] = cutout.months.start, cutout.months.stop - - metadata["files_prepared"] = {} - metadata["files_orig"] = {} - for yearmonth in cutout.coords["year-month"].to_index(): - fp = cutout.datasetfn(yearmonth) - with open(fp, "rb") as f: - metadata["files_orig"][str(Path(fp).relative_to(self._ref_path))] = ( - hashlib.sha256(f.read()).hexdigest() - ) + for d in dataset.catalog: + with open(d.path, "rb") as f: + metadata["files_orig"][ + str(Path(d.path).relative_to(self._ref_path)) + ] = hashlib.sha256(f.read()).hexdigest() return metadata @@ -329,9 +262,7 @@ def prepare(self, force: bool = False): (self._path / "nc4").mkdir(exist_ok=True, parents=True) self.metadata["files_prepared"] = {} - for fp in ( - self._prepare_dataset() if self.from_dataset else self._prepare_cutout() - ): + for fp in self._prepare_dataset(): with open(self._path / fp, "rb") as f: self.metadata["files_prepared"][fp] = hashlib.sha256( f.read() @@ -350,21 +281,6 @@ def _prepare_dataset(self) -> list: list: List of files. """ - @abc.abstractmethod - def _prepare_cutout(self) -> list: - """Prepare the model from a cutout. - - Returns: - list: List of files. - """ - - @property - def from_dataset(self) -> bool: - """Check if the model is from a dataset.""" - if "from_dataset" not in self.metadata: - return False - return self.metadata["from_dataset"] - @abc.abstractmethod def _estimate_dataset( self, @@ -389,30 +305,6 @@ def _estimate_dataset( xr.DataArray: Dataset with wind speed. """ - @abc.abstractmethod - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: - """Estimate the wind speed from a cutout. - - Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. - months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. - - Returns: - xr.DataArray: Dataset with wind speed. - """ - @property def files(self): if "files_prepared" not in self.metadata or "files_orig" not in self.metadata: diff --git a/src/geodata/model/wind/extrapolate.py b/src/geodata/model/wind/extrapolate.py index 37a4cb03..951be81c 100644 --- a/src/geodata/model/wind/extrapolate.py +++ b/src/geodata/model/wind/extrapolate.py @@ -191,41 +191,3 @@ def _estimate_dataset( result = alpha * np.log((height - ds["disph"]) / np.exp(-beta / alpha)) return result.drop_vars("coeff") # remove unnecessary coordinate - - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: - assert height > 0, "Height must be greater than 0." - - if months is None: - months = slice(1, 12) - - start_time = pd.Timestamp(year=years.start, month=months.start, day=1) - end_time = pd.Timestamp( - year=years.stop, month=months.stop, day=31, hour=23, minute=59, second=59 - ) - - ds = xr.open_mfdataset(self.files) - - if xs is None: - xs = ds.coords["x"] - if ys is None: - ys = ds.coords["y"] - - ds = ds.sel(x=xs, y=ys, time=slice(start_time, end_time)) - - if height in HEIGHTS.values() and use_real_data: - logger.info("Using real data for estimation at height %d", height) - return (ds[f"u{height}m"] ** 2 + ds[f"v{height}m"] ** 2) ** 0.5 - - alpha = ds["coeffs"][..., 0] - beta = ds["coeffs"][..., 1] - - result = alpha * np.log((height - ds["disph"]) / np.exp(-beta / alpha)) - return result.drop_vars("coeff") # remove unnecessary coordinate diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index 2e63e490..17d877f9 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -216,42 +216,3 @@ def _estimate_dataset( return xr.DataArray( spline_params(height), dims=params.dims, coords=params.coords ) - - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.Dataset: - params = xr.open_mfdataset(self.files).transpose("height", ...) - - if not (xs is None or ys is None): - params = params.sel(latitude=ys, longitude=xs) - - if not (years is None and months is None): - if months is None: - months = slice(1, 13) - params = params.sel( - valid_time=get_daterange(years, months), - ) - - if float(height) in LEVEL_TO_HEIGHT.values() and use_real_data: - params = params.sel(height=height) - return ( - ((params["u"] ** 2 + params["v"] ** 2) ** 0.5) - .drop("height") - .drop("model_level") - ) - - params = params[["c"]] - spline_params = sinterp.BSpline( - params.attrs.get("t"), params.get("c").values, k=3 - ) - - params = params.drop_dims("height") - return xr.DataArray( - spline_params(height), dims=params.dims, coords=params.coords - ) From 7fac81cd28ab4e14288580542bce190b3a66f0c0 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 17 Apr 2025 10:33:34 -0700 Subject: [PATCH 32/54] fix: conform model interface to new dataset interface --- src/geodata/model/_base.py | 54 ++++++++++++++++++++++++--------- src/geodata/model/wind/_base.py | 32 +------------------ 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 374dac2a..0472cdd2 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -43,7 +43,6 @@ class BaseModel(abc.ABC): metadata_keys: set[str] = { "name", "module", - "from_dataset", "years", "months", "files_orig", @@ -83,15 +82,17 @@ def __init__(self, source: BaseDataset, **kwargs): "files_orig", {} ) - if set(_source_files.keys()) != set( + if set(_source_files.keys()) - set( self.metadata.get("files_orig", {}).keys() ): logger.warning( - "Metadata file %s is outdated. Model will be re-prepared.", + "New dataset files have been downloaded since last model " + "preparation. Model will need to be re-prepared!", meta_path, ) - self._corrupt_metadata = True - self.metadata = self.extract_dataset_metadata(source) + self.metadata = self.extract_dataset_metadata( + source, self.metadata.get("files_prepared", {}) + ) except json.JSONDecodeError: logger.warning( @@ -155,7 +156,9 @@ def estimate( use_real_data=use_real_data, ) - def extract_dataset_metadata(self, dataset: BaseDataset) -> dict: + def extract_dataset_metadata( + self, dataset: BaseDataset, prepared: dict[str, str] | None = None + ) -> dict: if not dataset.downloaded: raise ValueError("The source dataset for this model is not prepared.") @@ -164,7 +167,6 @@ def extract_dataset_metadata(self, dataset: BaseDataset) -> dict: metadata = {} metadata["name"] = metadata["module"] = dataset.module - metadata["from_dataset"] = True if isinstance(dataset.years, slice): metadata["years"] = dataset.years.start, dataset.years.stop @@ -174,7 +176,7 @@ def extract_dataset_metadata(self, dataset: BaseDataset) -> dict: metadata["weather_data_config"] = dataset.weather_config # NOTE: file paths for estimation parameters will be added later in the prepare step - metadata["files_prepared"] = {} + metadata["files_prepared"] = {} if prepared is None else prepared metadata["files_orig"] = {} for d in dataset.catalog: with open(d.path, "rb") as f: @@ -204,6 +206,7 @@ def _check_prepared(self) -> bool: if not nc4_path.exists() or not meta_path.exists() or self._corrupt_metadata: return False + with open(meta_path, encoding="utf-8") as f: metadata_loaded = json.load(f) if set(metadata_loaded.keys()) != self.metadata_keys: @@ -216,10 +219,7 @@ def _check_prepared(self) -> bool: for fp in self.metadata["files_orig"]: fp_prepared = str(nc4_rel_path / Path(fp).with_suffix(".params.nc4")) - if ( - fp not in self.metadata["files_orig"] - or fp_prepared not in self.metadata["files_prepared"] - ): + if fp_prepared not in self.metadata["files_prepared"]: return False with open(self._ref_path / fp, "rb") as f: @@ -239,9 +239,11 @@ def _check_prepared(self) -> bool: != hashlib.sha256(f.read()).hexdigest() ): logger.warning( - "Parameter file %s in model has been modified since model creation. Model is not prepared!", + "Parameter file %s in model has been modified since model creation." + " This file will be re-prepared again.", fp_prepared, ) + del self.metadata["files_prepared"][fp_prepared] return False return True @@ -257,7 +259,6 @@ def prepare(self, force: bool = False): logger.info("The model is already prepared.") return - logger.info("Model not present in model directory, creating.") shutil.rmtree(self._path, ignore_errors=True) (self._path / "nc4").mkdir(exist_ok=True, parents=True) @@ -273,6 +274,31 @@ def prepare(self, force: bool = False): logger.info("Finished preparing model.") + @property + def files_orig(self): + """Get the original files. of the model.""" + + files_orig = [self._ref_path / p for p in self.metadata["files_orig"]] + return files_orig + + @property + def files_prepared(self): + """Get the prepared files of the model.""" + self._check_prepared() # Eliminate any corrupt files + + files_prepared = [self._path / p for p in self.metadata["files_prepared"]] + return files_prepared + + @property + def files_unprepared(self): + """Get the unprepared files of the model.""" + + original = set(self.metadata["files_orig"].keys()) + prepared = set(self.metadata["files_prepared"].keys()) + + unprepared = original - prepared + return list(unprepared) + @abc.abstractmethod def _prepare_dataset(self) -> list: """Prepare the model from a dataset. diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index 96e46622..60b1b649 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -69,7 +69,7 @@ def _prepare_dataset(self) -> list[tuple[str, Path]]: logger.info("Preparing the model from dataset.") prepared_files = [] - for file_path in tqdm(self.metadata["files_orig"], dynamic_ncols=True): + for file_path in tqdm(self.files_unprepared, dynamic_ncols=True): orig_ds_path: Path = self._ref_path / file_path ds = xr.open_dataset(orig_ds_path, chunks="auto") try: @@ -90,33 +90,3 @@ def _prepare_dataset(self) -> list[tuple[str, Path]]: prepared_files.append(str(ds_path.relative_to(self._path))) return prepared_files - - def _prepare_cutout(self) -> list[tuple[str, Path]]: - """Prepare the model from a cutout.""" - - logger.info("Preparing the model from cutout.") - prepared_files = [] - - for yearmonth in tqdm(self.source.coords["year-month"].to_index()): - orig_ds_path = Path(self.source.datasetfn(yearmonth)) - - ds = xr.open_dataset(orig_ds_path) - try: - ds = self._prepare_fn(ds) - except SystemError: - logger.warning( - "Could not compute wind speed of %s, possibly due to corrupt file.", - orig_ds_path.name, - ) - continue - - ds_path = orig_ds_path.relative_to(self._ref_path).with_suffix( - ".params.nc4" - ) - ds_path = self._path / "nc4" / ds_path - ds_path.parent.mkdir(parents=True, exist_ok=True) - ds.to_netcdf(ds_path) - - prepared_files.append(str(ds_path.relative_to(self._path))) - - return prepared_files From bbac2c456ce466254775cb60e51c880a1a483295 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 17 Apr 2025 10:46:53 -0700 Subject: [PATCH 33/54] fix: save file integrity check state --- src/geodata/datasets/_base.py | 22 ++++++++++++++++++++-- src/geodata/model/_base.py | 8 +++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 0529437b..dc664e01 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -66,7 +66,7 @@ def path(self): / f"{self.day:02d}.nc" ) - def check(self, integrity: bool = True): + def check(self, integrity: bool = False): """Check the presence of the file and its integrity. Args: @@ -207,6 +207,8 @@ def __init__( ) self.storage_root.mkdir(parents=True) + self._downloaded = False + self._extra_kwargs = kwargs self._extra_setup(**kwargs) @@ -249,7 +251,23 @@ def downloaded(self): a more comprehensive check is required. """ - return all((file.check() for file in self.catalog)) + if not self._downloaded: + self._downloaded = self._check_downloaded() + return self._downloaded + + def _check_downloaded(self): + # Check if the dataset is downloaded + for file in tqdm( + self.catalog, + unit="file", + dynamic_ncols=True, + desc="Checking Downloaded Files", + ): + if not file.check(): + logger.debug(f"{file.path} does not exist") + return False + + return True @abc.abstractmethod def _download_file(self, file: AtomicDataset): diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 0472cdd2..6c45393d 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -22,6 +22,7 @@ from typing import Optional import xarray as xr +from tqdm.auto import tqdm from ..config import model_dir from ..datasets._base import BaseDataset @@ -178,7 +179,12 @@ def extract_dataset_metadata( # NOTE: file paths for estimation parameters will be added later in the prepare step metadata["files_prepared"] = {} if prepared is None else prepared metadata["files_orig"] = {} - for d in dataset.catalog: + for d in tqdm( + dataset.catalog, + unit="file", + dynamic_ncols=True, + desc="Original Files Integrity Check", + ): with open(d.path, "rb") as f: metadata["files_orig"][ str(Path(d.path).relative_to(self._ref_path)) From 3c19d2c2f0a31049b2233c12a4fdda91295cdab9 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 17 Apr 2025 11:09:36 -0700 Subject: [PATCH 34/54] feat: support partial saving of parameters during interruption --- src/geodata/model/_base.py | 39 ++++++++++++++++++++------- src/geodata/model/wind/_base.py | 10 +++---- src/geodata/model/wind/interpolate.py | 2 +- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 6c45393d..6a2a17b4 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -18,6 +18,7 @@ import hashlib import json import shutil +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Optional @@ -25,7 +26,7 @@ from tqdm.auto import tqdm from ..config import model_dir -from ..datasets._base import BaseDataset +from ..datasets._base import AtomicDataset, BaseDataset from ..logging import logger from ..utils import NpEncoder @@ -179,16 +180,25 @@ def extract_dataset_metadata( # NOTE: file paths for estimation parameters will be added later in the prepare step metadata["files_prepared"] = {} if prepared is None else prepared metadata["files_orig"] = {} - for d in tqdm( - dataset.catalog, - unit="file", - dynamic_ncols=True, - desc="Original Files Integrity Check", - ): + + def compute_hash(d: AtomicDataset): with open(d.path, "rb") as f: - metadata["files_orig"][ - str(Path(d.path).relative_to(self._ref_path)) - ] = hashlib.sha256(f.read()).hexdigest() + return str(Path(d.path).relative_to(self._ref_path)), hashlib.sha256( + f.read() + ).hexdigest() + + with ThreadPoolExecutor() as executor: + results = list( + tqdm( + executor.map(compute_hash, dataset.catalog), + total=len(dataset.catalog), + unit="file", + dynamic_ncols=True, + desc="Original Files Integrity Check", + ) + ) + + metadata["files_orig"] = dict(results) return metadata @@ -278,6 +288,15 @@ def prepare(self, force: bool = False): with open(self._path / "meta.json", "w", encoding="utf-8") as f: json.dump(self.metadata, f, indent=4, cls=NpEncoder) + if len(set(self.metadata["files_orig"])) != len( + set(self.metadata["files_prepared"]) + ): + logger.warning( + "The number of original files and prepared files do not match. " + "This may indicate an issue with the preparation process. Partially prepared files were saved." + ) + return + logger.info("Finished preparing model.") @property diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index 60b1b649..fe8dfc48 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -69,17 +69,17 @@ def _prepare_dataset(self) -> list[tuple[str, Path]]: logger.info("Preparing the model from dataset.") prepared_files = [] + for file_path in tqdm(self.files_unprepared, dynamic_ncols=True): orig_ds_path: Path = self._ref_path / file_path ds = xr.open_dataset(orig_ds_path, chunks="auto") try: ds = self._prepare_fn(ds) - except SystemError: - logger.warning( - "Could not compute wind speed of %s, possibly due to corrupt file.", - orig_ds_path.name, + except Exception as e: + logger.error( + "Error preparing dataset %s: %s", orig_ds_path.name, str(e) ) - continue + return prepared_files ds_path: Path = ( self._path / "nc4" / Path(file_path).with_suffix(".params.nc4") diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index 17d877f9..b7e075e3 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -84,7 +84,7 @@ def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: from dask.array import map_blocks from dask.diagnostics import ProgressBar - logger.info("Computing interpolation coefficients using Dask.") + logger.debug("Computing interpolation coefficients using Dask.") if len(a.data.chunks[0]) > 1: a = a.chunk({dim: -1}) From 2c64f4fbd51264ceb12fd229f15632924c3404b1 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:18:46 -0700 Subject: [PATCH 35/54] refactor: new model preparation logic --- src/geodata/datasets/_base.py | 59 ++- src/geodata/model/_base.py | 550 ++++++++++++++++---------- src/geodata/model/wind/_base.py | 38 +- src/geodata/model/wind/extrapolate.py | 2 +- src/geodata/model/wind/interpolate.py | 4 +- src/geodata/utils.py | 30 +- 6 files changed, 440 insertions(+), 243 deletions(-) diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index dc664e01..fc71d54a 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -454,7 +454,45 @@ def catalog(self) -> list["AtomicDataset"]: return cat - def _monthly_catalog(self) -> list["AtomicDataset"]: + def get_monthly_catalog(self, year: int, month: int) -> list["AtomicDataset"]: + """Get the catalog for a specific month and year. + + Args: + year: The year of the file. + month: The month of the file. + """ + if not isinstance(year, int): + raise ValueError("year must be an integer") + if not isinstance(month, int): + raise ValueError("month must be an integer") + if not 1 <= month <= 12: + raise ValueError("month must be between 1 and 12") + + if not self.years.start <= year <= self.years.stop: + raise ValueError( + f"year must be between {self.years.start} and {self.years.stop}" + ) + + match self.frequency: + case "monthly": + return self._monthly_catalog(year, month) + case "daily": + return self._daily_catalog(year, month) + case _: + raise ValueError( + f"Invalid frequency {self.frequency} defined for this dataset." + ) + + def _monthly_catalog( + self, year: int | None = None, month: int | None = None + ) -> list["AtomicDataset"]: + if year is not None and month is not None: + return [AtomicDataset(self, year, month)] + if year is not None or month is not None: + raise ValueError( + "If one of year or month is specified, both must be specified." + ) + catalog = [] for year, month in itertools.product( @@ -465,7 +503,24 @@ def _monthly_catalog(self) -> list["AtomicDataset"]: return catalog - def _daily_catalog(self) -> list["AtomicDataset"]: + def _daily_catalog( + self, year: int | None = None, month: int | None = None + ) -> list["AtomicDataset"]: + if year is not None and month is not None: + return [ + AtomicDataset(self, year, month, day) + for day in range( + 1, + pd.Timestamp(f"{year}-{month}-1").days_in_month + 1 + if not self.testing + else 3, + ) + ] + if year is not None or month is not None: + raise ValueError( + "If one of year or month is specified, both must be specified." + ) + catalog = [] for year, month in itertools.product( diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 6a2a17b4..e4761b8f 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -17,18 +17,295 @@ import abc import hashlib import json +import os import shutil from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Optional, Self import xarray as xr from tqdm.auto import tqdm from ..config import model_dir -from ..datasets._base import AtomicDataset, BaseDataset +from ..datasets._base import BaseDataset from ..logging import logger -from ..utils import NpEncoder +from ..utils import check_hash + + +@dataclass +class ModelResult: + """Model result class. This class is used to store the result of a model. + It contains the year, month, reference path, model path, and the hashes of the + reference and model datasets. + + Args: + year (int): Year of the model. + month (int): Month of the model. + ref_path (Path): Path to the reference dataset. + path (Path): Path to the model dataset. + ref_hash (str): Hash of the reference dataset. + path_hash (str): Hash of the model dataset. + """ + + year: int + month: int + model: "BaseModel" + + _hashes: dict[str, str] = field(default_factory=dict) + _prepared: bool = False + + @property + def frequency(self) -> str: + """Frequency of the model.""" + return self.model.frequency + + @property + def module(self) -> str: + """Module of the model.""" + return self.model.source.module + + @property + def ref_path(self) -> Path: + """Path to the reference dataset.""" + return ( + self.model._ref_path + / self.model.source.weather_config + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def files(self) -> list[Path]: + """List of files in the model dataset. This could potentially include any + files that are not prepared yet.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Model files for {self.year:04d}-{self.month:02d} not found." + ) + + files = [] + for f in atomic_files: + p = f.path.relative_to(self.ref_path).with_stem(f.path.stem + ".params") + files.append(self.path / p) + return files + + @property + def ref_files(self) -> list[Path]: + """List of files in the reference dataset.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Reference files for {self.year:04d}-{self.month:02d} not found." + ) + + return [f.path for f in atomic_files if f.path.exists()] + + @property + def ref_params(self) -> dict[Path, Path]: + """Reference parameters of the model.""" + + ref_params = {} + for file in self.ref_files: + if file.name.endswith(".params.nc"): + ref_params[file] = self.path / file.name + else: + ref_params[file] = self.path / f"{file.stem}.params.nc" + return ref_params + + @property + def path(self) -> Path: + """Path to the model dataset.""" + return ( + model_dir + / self.module + / self.model.__class__.__name__ + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + if not self._prepared: + self._prepared = self._check_prepared() + return self._prepared + + def _check_prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + + assert self.path is not None, "The model saving path has not been set yet." + + if not (self.path / "meta.json").exists(): + logger.warning( + "Model %s-%s does not have metadata. Please prepare the model first.", + self.year, + self.month, + ) + return False + + match self.frequency: + case "daily": + with ThreadPoolExecutor( + max_workers=os.getenv("MAX_WORKERS") + ) as executor: + files = [(f, self._hashes.get(f.name)) for f in self.files] + results = list( + tqdm( + executor.map(lambda t: check_hash(*t), files), + total=len(files), + unit="file", + dynamic_ncols=True, + desc=f"Model Files Integrity Check {self.year:04d}-{self.month:02d}", + ) + ) + for file, (is_valid, hash_value) in zip(files, results): + if not is_valid: + logger.warning( + "File %s in model has been modified since model creation. Model is not prepared!", + file, + ) + return False + return True + case "monthly": + return check_hash(self.path / f"{self.month:02d}.params.nc")[0] + case _: + raise ValueError( + f"Frequency {self.frequency} is not supported. Supported frequencies are: daily, monthly." + ) + + def register(self, dataset: xr.Dataset): + """Register the model result with the dataset. + + Args: + dataset (xr.Dataset): Dataset to register. + """ + if not isinstance(dataset, xr.Dataset): + raise ValueError( + f"Dataset must be an xarray Dataset, but got {type(dataset)}." + ) + + match self.frequency: + case "daily": + day = dataset.get("valid_time").dt.day.values[0] + dataset.to_netcdf(self.path / f"{day:02d}.params.nc") + with open(self.path / f"{day:02d}.params.nc", "rb") as f: + self._hashes[f"{day:02d}.params.nc"] = hashlib.sha256( + f.read() + ).hexdigest() + + case "monthly": + dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc") + with open(self.path / f"{self.month:02d}.params.nc", "rb") as f: + self._hashes[f"{self.month:02d}.params.nc"] = hashlib.sha256( + f.read() + ).hexdigest() + + case _: + raise ValueError( + f"Frequency {self.frequency} is not supported. Supported frequencies are: daily, monthly." + ) + + @classmethod + def from_year_month(cls, model: "BaseModel", year: int, month: int) -> Self: + """Create an AtomicModel from year and month. + + Args: + model (BaseModel): BaseModel object. + year (int): Year of the model. + month (int): Month of the model. + + Returns: + AtomicModel: AtomicModel object. + """ + + if not (1 <= month <= 12): + raise ValueError(f"Month {month} is not valid. Must be between 1 and 12.") + if not (2000 <= year <= 2100): + raise ValueError( + f"Year {year} is not valid. Must be between 2000 and 2100." + ) + + path = ( + model_dir + / model.source.module + / model.__class__.__name__ + / f"{year:04d}" + / f"{month:02d}" + / "meta.json" + ) + + if not path.exists(): + return cls(model=model, year=year, month=month) + + with open(path, "r") as f: + data = json.load(f) + + if data["year"] != year or data["month"] != month: + raise ValueError( + f"Model year {data['year']} and month {data['month']} do not match {year} and {month}." + ) + + return cls.from_dict(data, model) + + def __repr__(self): + return f"AtomicModel(year={self.year}, month={self.month}, ref_path={self.ref_path}, path={self.path} {len(self.files)} / {len(self.ref_files)})" + + @classmethod + def from_dict(cls, data: dict, model: "BaseModel") -> Self: + """Create an AtomicModel from a dictionary. + + Args: + data (dict): Dictionary with the model data. + + Returns: + AtomicModel: AtomicModel object. + """ + + inst = cls(year=data["year"], month=data["month"], model=model) + + inst._hashes = data.get("hashes", {}) + inst.prepared + return inst + + def to_dict(self) -> dict: + """Convert the AtomicModel to a dictionary. + + Returns: + dict: Dictionary with the model data. + """ + + return { + "year": self.year, + "month": self.month, + "ref_path": str(self.ref_path), + "path": str(self.path), + "hashes": self._hashes, + "prepared": self.prepared, + } + + def dump(self): + """Dump the model result to a file. + + Returns: + dict: Dictionary with the model data. + """ + info = self.to_dict() + + with open(self.path / "meta.json", "w") as f: + json.dump(info, f, indent=4) + logger.info("Model result dumped to %s", self.path / "meta.json") class BaseModel(abc.ABC): @@ -65,66 +342,65 @@ def __init__(self, source: BaseDataset, **kwargs): self.source = source self._extra_kwargs = kwargs - self._corrupt_metadata = False self._prepared = False self._ref_path = model_dir.parent / self.source.module - self._path = ( - model_dir / self.type / self.__class__.__name__ / self.source.module - ) + self._results = self._prepare_results() - if (meta_path := self._path / "meta.json").exists(): - try: - with open(meta_path, encoding="utf_8") as f: - self.metadata = json.load(f) + def __repr__(self): + return f"Model(source={self.source}, type={self.type})" - # NOTE: Double-check if we have an updated dataset - # If we do, we need to re-prepare the model - _source_files = self.extract_dataset_metadata(source).get( - "files_orig", {} - ) + @property + def frequency(self) -> str: + """Frequency of the model.""" + return self.source.frequency - if set(_source_files.keys()) - set( - self.metadata.get("files_orig", {}).keys() - ): - logger.warning( - "New dataset files have been downloaded since last model " - "preparation. Model will need to be re-prepared!", - meta_path, - ) - self.metadata = self.extract_dataset_metadata( - source, self.metadata.get("files_prepared", {}) - ) + @property + @abc.abstractmethod + def type(self) -> str: + """Type of the model.""" - except json.JSONDecodeError: - logger.warning( - "Metadata file %s is corrupted. Model will be re-prepared.", - meta_path, - ) - self._corrupt_metadata = True - self.metadata = self.extract_dataset_metadata(source) + def _prepare_results(self) -> dict[int, dict[int, ModelResult]]: + """Prepare the results of the model. - else: - # NOTE: We only store metadata in transient fashion until preparation is done - self.metadata = self.extract_dataset_metadata(source) + Returns: + dict: Dictionary with the results of the model. + """ - def __repr__(self): - return f"Model(source={self.source}, type={self.type})" + years = list(range(self.source.years.start, self.source.years.stop + 1)) + months = list(range(self.source.months.start, self.source.months.stop + 1)) - @property - def name(self) -> str: - """Name of the model.""" - return f"{self.metadata['name']}_{self.type}" + results: dict[int, dict[int, ModelResult]] = {} + for year in years: + results[year] = {} + for month in months: + results[year][month] = ModelResult.from_year_month(self, year, month) + results[year][month].path.mkdir(parents=True, exist_ok=True) + + return results @property - def module(self) -> str: - """Module of the model.""" - return self.metadata["module"] + def results(self): + """Get the results of the model. + + Returns: + dict: Dictionary with the results of the model. + """ + return self._results @property - @abc.abstractmethod - def type(self) -> str: - """Type of the model.""" + def flattened_results(self) -> list[ModelResult]: + """Flatten the results of the model. + + Returns: + list: List of ModelResult objects. + """ + + return [ + self._results[year][month] + for year in self._results + for month in self._results[year] + ] def estimate( self, @@ -158,50 +434,6 @@ def estimate( use_real_data=use_real_data, ) - def extract_dataset_metadata( - self, dataset: BaseDataset, prepared: dict[str, str] | None = None - ) -> dict: - if not dataset.downloaded: - raise ValueError("The source dataset for this model is not prepared.") - - logger.info("Using dataset %s", dataset.module) - - metadata = {} - - metadata["name"] = metadata["module"] = dataset.module - - if isinstance(dataset.years, slice): - metadata["years"] = dataset.years.start, dataset.years.stop - if isinstance(dataset.months, slice): - metadata["months"] = dataset.months.start, dataset.months.stop - - metadata["weather_data_config"] = dataset.weather_config - - # NOTE: file paths for estimation parameters will be added later in the prepare step - metadata["files_prepared"] = {} if prepared is None else prepared - metadata["files_orig"] = {} - - def compute_hash(d: AtomicDataset): - with open(d.path, "rb") as f: - return str(Path(d.path).relative_to(self._ref_path)), hashlib.sha256( - f.read() - ).hexdigest() - - with ThreadPoolExecutor() as executor: - results = list( - tqdm( - executor.map(compute_hash, dataset.catalog), - total=len(dataset.catalog), - unit="file", - dynamic_ncols=True, - desc="Original Files Integrity Check", - ) - ) - - metadata["files_orig"] = dict(results) - - return metadata - @property def prepared(self) -> bool: """Check if the model is prepared. @@ -215,121 +447,49 @@ def prepared(self) -> bool: return self._prepared def _check_prepared(self) -> bool: - assert self._path is not None, "The model saving path has not been set yet." - - nc4_path: Path = self._path / "nc4" - meta_path: Path = self._path / "meta.json" - - if not nc4_path.exists() or not meta_path.exists() or self._corrupt_metadata: - return False - - with open(meta_path, encoding="utf-8") as f: - metadata_loaded = json.load(f) - if set(metadata_loaded.keys()) != self.metadata_keys: - return False - - if len(self.metadata["files_orig"]) != len(self.metadata["files_prepared"]): - return False - - nc4_rel_path = nc4_path.relative_to(self._path) - for fp in self.metadata["files_orig"]: - fp_prepared = str(nc4_rel_path / Path(fp).with_suffix(".params.nc4")) - - if fp_prepared not in self.metadata["files_prepared"]: - return False - - with open(self._ref_path / fp, "rb") as f: - if ( - self.metadata["files_orig"][fp] - != hashlib.sha256(f.read()).hexdigest() - ): - logger.warning( - "File %s in source dataset has been modified since model creation. Model is not prepared!", - fp, - ) + for year in self._results: + for month in self._results[year]: + if not self._results[year][month].prepared: return False - - with open(self._path / fp_prepared, "rb") as f: - if ( - self.metadata["files_prepared"][fp_prepared] - != hashlib.sha256(f.read()).hexdigest() - ): - logger.warning( - "Parameter file %s in model has been modified since model creation." - " This file will be re-prepared again.", - fp_prepared, - ) - del self.metadata["files_prepared"][fp_prepared] - return False - return True def prepare(self, force: bool = False): """Prepare the model. Args: - force (bool, optional): Force re-prepare the model. Defaults to False.""" + force (bool, optional): Force re-prepare the model. Defaults to False. + """ self._prepared = False # NOTE: force re-checking preparedness here if self.prepared and not force: logger.info("The model is already prepared.") return - shutil.rmtree(self._path, ignore_errors=True) - (self._path / "nc4").mkdir(exist_ok=True, parents=True) - - self.metadata["files_prepared"] = {} - for fp in self._prepare_dataset(): - with open(self._path / fp, "rb") as f: - self.metadata["files_prepared"][fp] = hashlib.sha256( - f.read() - ).hexdigest() - - with open(self._path / "meta.json", "w", encoding="utf-8") as f: - json.dump(self.metadata, f, indent=4, cls=NpEncoder) - - if len(set(self.metadata["files_orig"])) != len( - set(self.metadata["files_prepared"]) - ): - logger.warning( - "The number of original files and prepared files do not match. " - "This may indicate an issue with the preparation process. Partially prepared files were saved." - ) - return - - logger.info("Finished preparing model.") - - @property - def files_orig(self): - """Get the original files. of the model.""" - - files_orig = [self._ref_path / p for p in self.metadata["files_orig"]] - return files_orig + for result in self.flattened_results: + if not result.prepared: + shutil.rmtree(result.path, ignore_errors=True) + result.path.mkdir(parents=True, exist_ok=True) - @property - def files_prepared(self): - """Get the prepared files of the model.""" - self._check_prepared() # Eliminate any corrupt files - - files_prepared = [self._path / p for p in self.metadata["files_prepared"]] - return files_prepared - - @property - def files_unprepared(self): - """Get the unprepared files of the model.""" - - original = set(self.metadata["files_orig"].keys()) - prepared = set(self.metadata["files_prepared"].keys()) + for ref in tqdm(result.ref_params): + if not ref.exists(): + raise FileNotFoundError(f"Reference file {ref} does not exist.") + ref_ds = xr.open_dataset(ref) + prepared_ds = self._prepare_dataset(ref_ds) + ref_ds.close() + result.register(prepared_ds) - unprepared = original - prepared - return list(unprepared) + result.dump() + logger.info("Model prepared successfully.") @abc.abstractmethod - def _prepare_dataset(self) -> list: - """Prepare the model from a dataset. + def _prepare_dataset(self, source: xr.Dataset) -> xr.Dataset: + """Prepare the parameters of a specific source dataset file. + + Args: + source (xr.Dataset): Source dataset. Returns: - list: List of files. + xr.Dataset: Prepared parameter dataset. """ @abc.abstractmethod @@ -355,13 +515,3 @@ def _estimate_dataset( Returns: xr.DataArray: Dataset with wind speed. """ - - @property - def files(self): - if "files_prepared" not in self.metadata or "files_orig" not in self.metadata: - return [] - - files_prepared = [self._path / p for p in self.metadata["files_prepared"]] - files_orig = [self._ref_path / p for p in self.metadata["files_orig"]] - - return files_prepared + files_orig diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index fe8dfc48..849c8d51 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -39,13 +39,6 @@ >>> model.estimate(xs=slice(1, 2), ys=slice(1, 2), years=slice(2010, 2010), months=slice(1, 2)) """ -from pathlib import Path -from typing import Callable - -import xarray as xr -from tqdm.auto import tqdm - -from ...logging import logger from .._base import BaseModel HEIGHTS = {"u50m": 50, "u10m": 10, "u2m": 2} @@ -61,32 +54,3 @@ class WindBaseModel(BaseModel): """ type: str = "wind" - _prepare_fn: Callable[[xr.Dataset], xr.Dataset] - - def _prepare_dataset(self) -> list[tuple[str, Path]]: - """Prepare the model from a dataset.""" - - logger.info("Preparing the model from dataset.") - - prepared_files = [] - - for file_path in tqdm(self.files_unprepared, dynamic_ncols=True): - orig_ds_path: Path = self._ref_path / file_path - ds = xr.open_dataset(orig_ds_path, chunks="auto") - try: - ds = self._prepare_fn(ds) - except Exception as e: - logger.error( - "Error preparing dataset %s: %s", orig_ds_path.name, str(e) - ) - return prepared_files - - ds_path: Path = ( - self._path / "nc4" / Path(file_path).with_suffix(".params.nc4") - ) - ds_path.parent.mkdir(parents=True, exist_ok=True) - ds.to_netcdf(ds_path) - - prepared_files.append(str(ds_path.relative_to(self._path))) - - return prepared_files diff --git a/src/geodata/model/wind/extrapolate.py b/src/geodata/model/wind/extrapolate.py index 951be81c..17ac008e 100644 --- a/src/geodata/model/wind/extrapolate.py +++ b/src/geodata/model/wind/extrapolate.py @@ -93,7 +93,7 @@ class WindExtrapolationModel(WindBaseModel): SUPPORTED_WEATHER_DATA_CONFIGS = {"slv_flux_hourly"} - def _prepare_fn( + def _prepare_dataset( self, ds: xr.Dataset, compute_lml: bool = True, diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index b7e075e3..0cd2e7be 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -1,4 +1,4 @@ -# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -140,7 +140,7 @@ class WindInterpolationModel(WindBaseModel): SUPPORTED_WEATHER_DATA_CONFIGS = {"wind_3d_hourly"} - def _prepare_fn( + def _prepare_dataset( self, ds: xr.Dataset, half_precision: bool = True, diff --git a/src/geodata/utils.py b/src/geodata/utils.py index 42640631..951d996a 100644 --- a/src/geodata/utils.py +++ b/src/geodata/utils.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,7 +14,9 @@ # along with this program. If not, see . +import hashlib import json +from pathlib import Path import numpy as np import pandas as pd @@ -87,3 +89,29 @@ def ensure_slice(obj: slice | list): return obj else: raise TypeError("Input must be a slice or a list.") + + +def check_hash(file: Path, saved_hash: str | None = None) -> tuple[bool, str]: + """Check if the hash of a file matches the given hash. + + Args: + file (Path): The path to the file. + saved_hash (str | None): The hash to compare against. If None, the function will compute the hash of the file. + If provided, the function will compare the computed hash with this value. + If the hashes match, the function will return True and the computed hash. + If the hashes do not match, the function will return False and the computed hash. + + Returns: + tuple[bool, str]: A tuple containing a boolean indicating if the hash matches and the computed hash. + """ + + if not file.exists(): + return False, "" + + with open(file, "rb") as f: + computed_hash = hashlib.sha256(f.read()).hexdigest() + + if saved_hash is None: + return True, computed_hash + else: + return computed_hash == saved_hash, computed_hash From 0f0e79140096cedd4f0f44ebe42eec83bea888cb Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 5 May 2025 14:46:03 -0700 Subject: [PATCH 36/54] refactor: update proper type annotation for `BaseDataset` class --- src/geodata/datasets/__init__.py | 4 +- src/geodata/datasets/_base.py | 9 +- src/geodata/datasets/_era5.py | 599 ------------------------------ src/geodata/datasets/_merra2.py | 612 ------------------------------- 4 files changed, 8 insertions(+), 1216 deletions(-) delete mode 100644 src/geodata/datasets/_era5.py delete mode 100644 src/geodata/datasets/_merra2.py diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index 46017244..80023216 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -14,10 +14,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + from . import era5, merra2 +from ._base import DatasetType from ._base import _registry as registry -__all__ = ["era5", "merra2", "registry", "register_hrrr"] +__all__ = ["era5", "merra2", "registry", "register_hrrr", "DatasetType"] def register_hrrr(): diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index fc71d54a..3e265670 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -20,7 +20,7 @@ import logging from collections.abc import Sequence from pathlib import Path -from typing import Literal, Type +from typing import Literal, Type, TypeVar import pandas as pd import xarray as xr @@ -30,8 +30,9 @@ from ..types import BoundRange, CoordRange, DateRange logger = logging.getLogger(__name__) +DatasetType = TypeVar("DatasetType", bound="BaseDataset") -_registry: dict[str, Type["BaseDataset"]] = {} +_registry: dict[str, DatasetType] = {} @dataclasses.dataclass @@ -41,7 +42,7 @@ class AtomicDataset: and integrity checking of these datasets. """ - dataset: "BaseDataset" + dataset: DatasetType year: int month: int day: int | None = None @@ -50,7 +51,7 @@ class AtomicDataset: spinup: bool | None = None def __post_init__(self): - if not isinstance(self.dataset, BaseDataset): + if not isinstance(self.dataset, DatasetType): raise ValueError("dataset must be an instance of BaseDataset") @property diff --git a/src/geodata/datasets/_era5.py b/src/geodata/datasets/_era5.py deleted file mode 100644 index 6dd23b42..00000000 --- a/src/geodata/datasets/_era5.py +++ /dev/null @@ -1,599 +0,0 @@ -# Copyright 2016-2017 Jonas Hoersch (FIAS), Tom Brown (FIAS), Markus Schlott (FIAS) -# Copyright 2022-2023 Xiqiang Liu - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -GEODATA - -Geospatial Data Collection and "Pre-Analysis" Tools -""" - -import calendar -import glob -import logging -import os -import tempfile -import zipfile -from pathlib import Path -from typing import Iterable - -import numpy as np -import xarray as xr - -from ..config import era5_dir - -logger = logging.getLogger(__name__) -datadir = era5_dir - -try: - import cdsapi - - has_cdsapi = True -except ImportError: - has_cdsapi = False - -# Model and Projection Settings -projection = "latlong" -L137_LEVELS = range(131, 138) - - -def _rename_and_clean_coords(ds: xr.Dataset, add_lon_lat: bool = True): - """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' - - Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. - - Args: - ds (xarray.Dataset): Dataset to rename - add_lon_lat (bool, optional): Add lon/lat columns. Defaults to True. - - Returns: - xarray.Dataset: Dataset with renamed coordinates - """ - - # Rename latitude / lat -> y, longitude / lon -> x - if "latitude" in list(ds.coords): - ds = ds.rename({"latitude": "y"}) - if "longitude" in list(ds.coords): - ds = ds.rename({"longitude": "x"}) - if "lat" in list(ds.coords): - ds = ds.rename({"lat": "y"}) - if "lon" in list(ds.coords): - ds = ds.rename({"lon": "x"}) - - if add_lon_lat: - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - - return ds - - -def api_complete( - toDownload: Iterable, - bounds: Iterable, - download_vars: Iterable, - product: str, - product_type: str, - downloadedFiles: list, -): - """Sample request: - - c.retrieve('reanalysis-era5-complete', { - 'date' : '20130101', - 'levelist': '1/10/100/137', - 'levtype' : 'ml', - 'param' : '130, # Full information at https://apps.ecmwf.int/codes/grib/param-db/ - 'stream' : 'oper', # Denotes ERA5. Ensemble members are selected by 'enda' - 'time' : '00/to/23/by/6', - 'type' : 'an', - 'grid' : '1.0/1.0', - 'format' : 'netcdf', - }, 'save_path.nc') # Output file. Adapt as you wish. - """ - - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - filepath = Path(f[1]) - filepath.parent.mkdir(parents=True, exist_ok=True) - - query_year = str(f[2]) - query_month = f"{f[3]:02d}" - - full_request = { - "date": f"{query_year}{query_month}01/to/{query_year}{query_month}{calendar.monthrange(f[2], f[3])[1]}", - "levelist": "/".join(str(i) for i in L137_LEVELS), - "levtype": "ml", - "param": "/".join(str(var) for var in download_vars), - "stream": "oper", - "time": "00/to/23", - "type": "an", - "grid": "0.25/0.25", # NOTE: We want the highest resolution possible - "format": "netcdf", - } - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["param"]), - f, - ) - - if not bounds: - full_result.download(f[1]) - else: - # NOTE: Raw MARS request doesn't support bounding box, so we need to - # subset the data after downloading - with tempfile.NamedTemporaryFile(suffix=".nc") as tmpfile: - full_result.download(tmpfile.name) - with xr.open_dataset(tmpfile.name, chunks="auto") as ds: - ds = ds.sel( - longitude=slice(*sorted([bounds[0], bounds[2]])), - latitude=slice( - *sorted([bounds[1], bounds[3]], reverse=True) - ), - ) - ds.to_netcdf(f[1]) - - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def api_hourly_era5( - toDownload: list, bounds, download_vars, product, product_type, downloadedFiles -): - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - print(f) - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - - ## for each file in self.todownload - need to then reextract year month in order to make query - query_year = str(f[2]) - query_month = str(f[3]) if len(str(f[3])) == 2 else "0" + str(f[3]) - - # 2. Full data file - full_request = { - "product_type": product_type, - "format": "netcdf", - "year": query_year, - "month": query_month, - "day": [f"{d:02d}" for d in range(1, 32)], - "time": [f"{h:02d}:00" for h in range(0, 24)], - "variable": download_vars, - } - - if bounds is not None: - # NOTE: cdsapi uses (long2, lat2, long1, lat1) format - full_request["area"] = bounds[::-1] - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["variable"]), - f, - ) - - if full_result.content_type == "application/zip": - logger.info( - "Multiple files found with request. Additional unzipping/preprocessing needed." - ) - - with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) - with zipfile.ZipFile( - os.path.join(tempdir, "download.zip"), "r" - ) as zip_ref: - zip_ref.extractall(tempdir) - - with xr.open_mfdataset( - [ - os.path.join(tempdir, f) - for f in os.listdir(tempdir) - if f.endswith(".nc") - ] - ) as ds: - ds.to_netcdf(f[1]) - - logger.info("Preprocessing complete with zipfile") - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - else: - full_result.download(f[1]) - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def api_monthly_era5( - toDownload, bounds, download_vars, product, product_type, downloadedFiles -): - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - print(f) - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - - ## for each file in self.todownload - need to then reextract year month in order to make query - query_year = str(f[2]) - query_month = str(f[3]) if len(str(f[3])) == 2 else "0" + str(f[3]) - - # 2. Full data file - full_request = { - "product_type": product_type, - "format": "netcdf", - "year": query_year, - "month": query_month, - "time": "00:00", - "variable": download_vars, - } - - if bounds is not None: - # cdsapi uses (long2, lat2, long1, lat1) format - full_request["area"] = bounds[::-1] - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["variable"]), - f, - ) - - if full_result.content_type == "application/zip": - logger.info( - "Multiple files found with request. Additional unzipping/preprocessing needed." - ) - - with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) - with zipfile.ZipFile( - os.path.join(tempdir, "download.zip"), "r" - ) as zip_ref: - zip_ref.extractall(tempdir) - - with xr.open_mfdataset( - [ - os.path.join(tempdir, f) - for f in os.listdir(tempdir) - if f.endswith(".nc") - ] - ) as ds: - ds.to_netcdf(f[1]) - - logger.info("Preprocessing complete with zipfile") - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - else: - full_result.download(f[1]) - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def _add_height(ds): - """Convert geopotential 'z' to geopotential height following [1] - - References - ---------- - [1] ERA5: surface elevation and orography, retrieved: 10.02.2019 - https://confluence.ecmwf.int/display/CKB/ERA5%3A+surface+elevation+and+orography - - """ - g0 = 9.80665 - z = ds["z"] - if "time" in z.coords: - z = z.isel(time=0, drop=True) - ds["height"] = z / g0 - ds = ds.drop("z") - return ds - - -def convert_and_subset_lons_lats_era5(ds, xs, ys): - # Rename geographic dimensions to x,y - # Subset x,y according to xs, ys (subset_x_y_era5) - - # Rename lat and lon - ds = _rename_and_clean_coords(ds) - - # Longitudes should go from -180. to +180. - if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): - ds = xr.concat( - [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" - ) - ds = ds.assign_coords( - x=np.where( - ds.coords["x"].values <= 180, - ds.coords["x"].values, - ds.coords["x"].values - 360.0, - ) - ) - # Subset x and y - ds = subset_x_y_era5(ds, xs, ys) - - return ds - - -def subset_x_y_era5(ds, xs, ys): - # Subset x,y according to xs, ys - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(y=ys) - ds = ds.sel(x=xs) - - return ds - - -def prepare_meta_era5(xs, ys, year, month, template, module, **kwargs): - # Reference of the quantities - # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation - # Geopotential is aka Orography in the CDS: - # https://confluence.ecmwf.int/pages/viewpage.action?pageId=78296105 - - fns = glob.iglob(template.format(year=year, month=month)) - - try: - with xr.open_mfdataset(fns, combine="by_coords") as ds0: - ds = ds0.coords.to_dataset() - ds = convert_and_subset_lons_lats_era5(ds, xs, ys) - meta = ds.load() - except Exception as e: - logger.exception("Error when preparing for cutout: %s", e.args[0]) - raise e - return meta - - -def prepare_month_era5(fn, year, month, xs, ys): - # Reference of the quantities - # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation - # (shortName) | (name) | (paramId) - # tisr | TOA incident solar radiation | 212 - # ssrd | Surface Solar Rad Downwards | 169 - # ssr | Surface net Solar Radiation | 176 - # fdir | Total sky direct solar radiation at surface | 228021 - # ro | Runoff | 205 - # 2t | 2 metre temperature | 167 - # sp | Surface pressure | 134 - # stl4 | Soil temperature level 4 | 236 - # fsr | Forecast surface roughnes | 244 - - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = _add_height(ds) - ds = subset_x_y_era5(ds, xs, ys) - - ds = ds.rename({"fdir": "influx_direct", "tisr": "influx_toa"}) - with np.errstate(divide="ignore", invalid="ignore"): - ds["albedo"] = ( - ((ds["ssrd"] - ds["ssr"]) / ds["ssrd"]) - .fillna(0.0) - .assign_attrs(units="(0 - 1)", long_name="Albedo") - ) - ds["influx_diffuse"] = (ds["ssrd"] - ds["influx_direct"]).assign_attrs( - units="J m**-2", long_name="Surface diffuse solar radiation downwards" - ) - ds = ds.drop(["ssrd", "ssr"]) - - # Convert from energy to power J m**-2 -> W m**-2 and clip negative fluxes - for a in ("influx_direct", "influx_diffuse", "influx_toa"): - ds[a] = ds[a].clip(min=0.0) / (60.0 * 60.0) - ds[a].attrs["units"] = "W m**-2" - - ds["wnd100m"] = np.sqrt(ds["u100"] ** 2 + ds["v100"] ** 2).assign_attrs( - units=ds["u100"].attrs["units"], long_name="100 metre wind speed" - ) - ds = ds.drop(["u100", "v100"]) - - ds = ds.rename( - { - "ro": "runoff", - "t2m": "temperature", - "sp": "pressure", - "stl4": "soil temperature", - "fsr": "roughness", - } - ) - - # New ERA5 format for hourly datasets - # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 - # TODO: We can remove this if we refactor geodata's convert module in the future - if "valid_time" in ds.coords: - ds = ds.rename({"valid_time": "time"}) - - ds["runoff"] = ds["runoff"].clip(min=0.0) - yield (year, month), ds - - -def tasks_monthly_era5(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info(list(yearmonths)) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format(year=year, month=month), - ) - for year, month in yearmonths - ] - - -def prepare_3d_era5(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = subset_x_y_era5(ds, xs, ys) - - # New ERA5 format for hourly datasets - # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 - # TODO: We can remove this if we refactor geodata's convert module in the future - if "valid_time" in ds.coords: - ds = ds.rename({"valid_time": "time"}) - - yield (year, month), ds - - -weather_data_config = { - "wind_solar_hourly": dict( - api_func=api_hourly_era5, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_month_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_hourly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_hourly.nc"), - product="reanalysis-era5-single-levels", - product_type="reanalysis", - keywords=[ - "100m_u_component_of_wind", - "100m_v_component_of_wind", - "2m_temperature", - "runoff", - "soil_temperature_level_4", - "surface_net_solar_radiation", - "surface_pressure", - "surface_solar_radiation_downwards", - "toa_incident_solar_radiation", - "total_sky_direct_solar_radiation_at_surface", - "forecast_surface_roughness", - "geopotential", - ], - variables=[ - "u100", - "v100", - "t2m", - "ro", - "stl4", - "ssr", - "sp", - "ssrd", - "tisr", - "fdir", - "fsr", - "z", - ], - ), - "wind_3d_hourly": dict( - api_func=api_complete, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_3d_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_3d_hourly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_3d_hourly.nc"), - product="reanalysis-era5-complete", - product_type="reanalysis", - keywords=[131, 132], - variables=["u", "v"], - ), - "wind_solar_monthly": dict( - api_func=api_monthly_era5, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_month_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_monthly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_monthly.nc"), - product="reanalysis-era5-single-levels-monthly-means", - product_type="monthly_averaged_reanalysis", - keywords=[ - "100m_u_component_of_wind", - "100m_v_component_of_wind", - "2m_temperature", - "runoff", - "soil_temperature_level_4", - "surface_net_solar_radiation", - "surface_pressure", - "surface_solar_radiation_downwards", - "toa_incident_solar_radiation", - "total_sky_direct_solar_radiation_at_surface", - "forecast_surface_roughness", - "geopotential", - ], - variables=[ - "u100", - "v100", - "t2m", - "ro", - "stl4", - "ssr", - "sp", - "ssrd", - "tisr", - "fdir", - "fsr", - "z", - ], - ), -} - -# No separate files for each day (would be coded in weather_data_config list, see merra2.py) -daily_files = False - -# Latitude direction stored -# South to north = True -# North to south = False -lat_direction = False - -# Spinup variable (necessary for MERRA) -spinup_var = False diff --git a/src/geodata/datasets/_merra2.py b/src/geodata/datasets/_merra2.py deleted file mode 100644 index 014b09f1..00000000 --- a/src/geodata/datasets/_merra2.py +++ /dev/null @@ -1,612 +0,0 @@ -# Copyright 2020 Michael Davidson (UCSD), William Honaker. - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -GEODATA - -Geospatial Data Collection and "Pre-Analysis" Tools -""" - -import glob -import os -from calendar import monthrange -from tempfile import mkstemp - -import numpy as np -import requests -import xarray as xr -from requests.exceptions import HTTPError -from tqdm.contrib.logging import tqdm_logging_redirect - -from ..config import merra2_dir -from ..logging import logger - -datadir = merra2_dir - -# Model and Projection Settings -projection = "latlong" - - -def convert_and_subset_lons_lats_merra2(ds, xs, ys): - # Rename geographic dimensions to x,y - # Subset x,y according to xs, ys - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(lat=ys) - - # Lons should go from -180. to +180. - if len(ds.coords["lon"].sel(lon=slice(xs.start + 360.0, xs.stop + 360.0))): - ds = xr.concat( - [ds.sel(lon=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(lon=xs)], - dim="lon", - ) - ds = ds.assign_coords( - lon=np.where( - ds.coords["lon"].values <= 180, - ds.coords["lon"].values, - ds.coords["lon"].values - 360.0, - ) - ) - else: - ds = ds.sel(lon=xs) - - ds = ds.rename({"lon": "x", "lat": "y"}) - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - return ds - - -def subset_x_y_merra2(ds, xs, ys): - # Subset x,y according to xs, ys - # Assumes convert_and_subset_lons_lats_merra2 already run - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(y=ys) - ds = ds.sel(x=xs) - - return ds - - -def _rename_and_clean_coords(ds, add_lon_lat=True): - """Rename 'longitude' and 'latitude' columns to 'x' and 'y' - - Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. - """ - - ds = ds.rename({"lon": "x", "lat": "y"}) - if add_lon_lat: - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - return ds - - -def api_merra2(toDownload, fileGranularity, downloadedFiles): - if len(toDownload) == 0: - logger.info("All MERRA2 files for this dataset have been downloaded.") - else: - multi = bool(fileGranularity in ("daily_multiple", "monthly_multiple")) - - total = len(toDownload) * (len(toDownload[0]) - 2) if multi else len(toDownload) - with tqdm_logging_redirect( - loggers=[logger], total=total, dynamic_ncols=True - ) as pbar: - for f in toDownload: - error_files = [] - if multi: - fd, target = mkstemp(suffix=".nc4") - else: - fd = 0 - target = f[1] - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - logger.info("Preparing API calls for %s", f[1]) - logger.info("Making request to %s", f[2]) - result = requests.get(f[2], timeout=30) - try: - result.raise_for_status() - with open(target, "wb") as fout: - fout.write(result.content) - except HTTPError as http_err: - logger.warning("HTTP error occurred: %s", http_err) # Python 3.6 - error_files.append(f[1]) - except Exception as err: # pylint: disable=broad-except - logger.warning("Other error occurred: %s", err) - error_files.append(f[1]) - pbar.update() - - if multi: - # ds_main = xr.open_dataset(target) - - temp_files = [[fd, target]] - for k in range(3, len(f)): - logger.info("Making request to %s", f[k]) - result = requests.get(f[k], timeout=30) - fd_temp, target_temp = mkstemp(suffix=".nc4") - temp_files.append([fd_temp, target_temp]) - try: - result.raise_for_status() - with open(target_temp, "wb") as fout: - fout.write(result.content) - except HTTPError as http_err: - logger.warning( - "HTTP error occurred: %s", http_err - ) # Python 3.6 - error_files.append(f[k]) - except Exception as err: # pylint: disable=broad-except - logger.warning("Other error occurred: %s", err) - error_files.append(f[k]) - # ds_toadd = xr.open_dataset(target_temp) - # ds_main = xr.merge([ds_main, ds_toadd]) - # os.close(fd_temp) - # os.unlink(target_temp) - pbar.update() - - ds_main = xr.open_mfdataset( - [fn[1] for fn in temp_files], combine="by_coords" - ) - ds_main.to_netcdf(f[1]) - ds_main.close() - # ds_toadd.close() # close last xr open file - - # close and clear temp files - # os.close(fd) - # os.unlink(target) - for tf in temp_files: - os.close(tf[0]) - os.unlink(tf[1]) - - if len(error_files) > 0: - logger.warning("Unsuccessful download for %s", error_files) - else: - logger.info("Successfully downloaded data for %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def prepare_meta_merra2(xs, ys, year, month, template, module, **params): - # Load dataset into metadata - - # fn = next(glob.iglob(template.format(year=year, month=month))) - # with xr.open_dataset(fn) as ds: - # ds = ds.coords.to_dataset() - # ds = convert_and_subset_lons_lats_merra2(ds, xs, ys) - # meta = ds.load() - - # Set spinup variable (see MERRA2 documentation, p. 13) - spinup = spinup_year(year, month) - - fns = glob.iglob(template.format(year=year, month=month, spinup=spinup)) - with xr.open_mfdataset(fns, combine="by_coords") as ds: - ds = ds.coords.to_dataset() - ds = convert_and_subset_lons_lats_merra2(ds, xs, ys) - meta = ds.load() - - return meta - - -def prepare_month_surface_flux(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - # some variable renaming - try: - # z0m=roughness - # wind variables not in wndXXm format - ds = ds.rename({"z0m": "roughness"}) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - - ds["wndlml"] = np.sqrt(ds["ulml"] ** 2 + ds["vlml"] ** 2).assign_attrs( - units=ds["ulml"].attrs["units"], long_name="LML wind speed" - ) - if "tlml" in list(ds.data_vars): - ds["temperature"] = ds["tlml"] - - yield (year, month), ds - - -def prepare_month_aerosol(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = subset_x_y_merra2(ds, xs, ys) - yield (year, month), ds - - -def prepare_dailymeans_surface_flux(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - # some variable renaming - try: - # z0m=roughness - # wind variables not in wndXXm format - ds = ds.rename({"t2mmean": "temperature", "tprecmax": "precipitation"}) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - - # ['HOURNORAIN', 'T2MMAX', 'T2MMEAN', 'T2MMIN', 'TPRECMAX'] - - yield (year, month), ds - - -def prepare_slv_radiation(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - try: - ds = ds.rename( - { - "albedo": "albedo", - "swgdn": "influx", - "swtdn": "influx_toa", - "t2m": "temperature", - } - ) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - yield (year, month), ds - - -## TODO def prepare_month_radiation -# with np.errstate(divide='ignore', invalid='ignore'): -# ds['albedo'] = (((ds['ssrd'] - ds['ssr'])/ds['ssrd']).fillna(0.) -# .assign_attrs(units='(0 - 1)', long_name='Albedo')) -# ds['influx_diffuse'] = ((ds['ssrd'] - ds['influx_direct']) -# .assign_attrs(units='J m**-2', -# long_name='Surface diffuse solar radiation downwards')) -# ds = ds.drop(['ssrd', 'ssr']) -# -# # Convert from energy to power J m**-2 -> W m**-2 and clip negative fluxes -# for a in ('influx_direct', 'influx_diffuse', 'influx_toa'): -# ds[a] = ds[a].clip(min=0.) / (60.*60.) -# ds[a].attrs['units'] = 'W m**-2' - - -def tasks_daily_merra2(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info( - [ - (year, month, day) - for year, month in yearmonths - for day in range(1, monthrange(year, month)[1] + 1, 1) - ] - ) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format( - year=year, month=month, day=day, spinup=spinup_year(year, month) - ), - ) - for year, month in yearmonths - for day in range(1, monthrange(year, month)[1] + 1, 1) - ] - - -def tasks_monthly_merra2(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info([(year, month) for year, month in yearmonths]) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format(year=year, month=month, spinup=spinup_year(year, month)), - ) - for year, month in yearmonths - ] - - -weather_data_config = { - "surface_flux_hourly": dict( - api_func=api_merra2, - file_granularity="daily", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_flx_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - url_opendap="https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ), - "slv_flux_hourly": dict( - api_func=api_merra2, - file_granularity="daily_multiple", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_slv_flx_Nx.*.nc4" - ), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - url_opendap=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - "u2m", - "v2m", - "u10m", - "v10m", - "u50m", - "v50m", - ], - variables_list=[ - [ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ["u2m", "v2m", "u10m", "v10m", "u50m", "v50m"], - ], - ), - "surface_flux_monthly": dict( - api_func=api_merra2, - file_granularity="monthly", - tasks_func=tasks_monthly_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join(merra2_dir, "{year}/MERRA2_*.tavgM_2d_flx_Nx.*.nc4"), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXFLX.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4", - fn=os.path.join( - merra2_dir, "{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4" - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ), - "surface_flux_dailymeans": dict( - api_func=api_merra2, - file_granularity="dailymeans", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_dailymeans_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.statD_2d_slv_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2SDNXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["hournorain", "tprecmax", "t2mmax", "t2mmean", "t2mmin"], - ), - "slv_radiation_hourly": dict( - api_func=api_merra2, - file_granularity="daily_multiple", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_slv_radiation, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_slv_rad_Nx.*.nc4" - ), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - url_opendap=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["albedo", "swgdn", "swtdn", "t2m"], - variables_list=[["t2m"], ["albedo", "swgdn", "swtdn"]], - ), - "slv_radiation_monthly": dict( - api_func=api_merra2, - file_granularity="monthly_multiple", - tasks_func=tasks_monthly_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_slv_radiation, - template=os.path.join(merra2_dir, "{year}/MERRA2_*.tavgM_2d_slv_rad_Nx.*.nc4"), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXSLV.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_slv_Nx.{year}{month:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXRAD.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_rad_Nx.{year}{month:0>2}.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/MERRA2_{spinup}.tavgM_2d_slv_rad_Nx.{year}{month:0>2}.nc4", - ), - variables=["albedo", "swgdn", "swtdn", "t2m"], - ), - "surface_aerosol_hourly": dict( - api_func=api_merra2, - file_granularity="daily", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_aerosol, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_aer_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXAER.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["bcsmass", "dusmass25", "ocsmass", "so4smass", "sssmass25"], - ), -} - -# os.path.join(merra2_dir, '{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4'), - -# list of routines in weather_data_config to download wind data -wind_files = ["surface_flux"] - -# TODO: same for solar -solar_files = [] - -## Whatever is calling this needs to be directed to the correct weather config instead -# meta_data_config = dict(prepare_func=prepare_meta_merra2, -# template=os.path.join(merra2_dir, '{year}/{month:0>2}/MERRA2_*.tavg1_2d_flx_Nx.*.nc4')) - -# Separate files for each day (coded in weather_data_config list) -# daily_files = True # needs to be specified somewhere else - -# Latitude stored south to north (ie forward, = True) or north to south -lat_direction = True - -# Spinup variable -spinup_var = True - - -def spinup_year(year, month): - if year >= 1980 and year < 1992: - spinup = "100" - elif year >= 1992 and year < 2001: - spinup = "200" - elif year >= 2001 and year < 2011: - spinup = "300" - elif year >= 2011 and year < 2020: - spinup = "400" - elif year == 2020 and month == 9: - spinup = "401" - else: - spinup = "400" - - return spinup From 40a8d46b0151f229789baf238774bd8b85f3c357 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sun, 11 May 2025 15:57:09 -0700 Subject: [PATCH 37/54] feat: implement model result classes for daily and monthly computations refactor: update dataset validation in AtomicDataset and BaseModel fix: enhance wind interpolation logic and dataset rechunking utility --- src/geodata/datasets/_base.py | 2 +- src/geodata/model/_base.py | 396 ++++---------------------- src/geodata/model/results/__init__.py | 23 ++ src/geodata/model/results/_base.py | 252 ++++++++++++++++ src/geodata/model/results/daily.py | 119 ++++++++ src/geodata/model/results/monthly.py | 51 ++++ src/geodata/model/wind/interpolate.py | 71 ++--- src/geodata/utils.py | 65 +++++ 8 files changed, 595 insertions(+), 384 deletions(-) create mode 100644 src/geodata/model/results/__init__.py create mode 100644 src/geodata/model/results/_base.py create mode 100644 src/geodata/model/results/daily.py create mode 100644 src/geodata/model/results/monthly.py diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 3e265670..4deed3a7 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -51,7 +51,7 @@ class AtomicDataset: spinup: bool | None = None def __post_init__(self): - if not isinstance(self.dataset, DatasetType): + if not isinstance(self.dataset, BaseDataset): raise ValueError("dataset must be an instance of BaseDataset") @property diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index e4761b8f..1decabb1 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -15,14 +15,8 @@ import abc -import hashlib -import json -import os import shutil -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional, Self +from typing import Optional import xarray as xr from tqdm.auto import tqdm @@ -30,282 +24,7 @@ from ..config import model_dir from ..datasets._base import BaseDataset from ..logging import logger -from ..utils import check_hash - - -@dataclass -class ModelResult: - """Model result class. This class is used to store the result of a model. - It contains the year, month, reference path, model path, and the hashes of the - reference and model datasets. - - Args: - year (int): Year of the model. - month (int): Month of the model. - ref_path (Path): Path to the reference dataset. - path (Path): Path to the model dataset. - ref_hash (str): Hash of the reference dataset. - path_hash (str): Hash of the model dataset. - """ - - year: int - month: int - model: "BaseModel" - - _hashes: dict[str, str] = field(default_factory=dict) - _prepared: bool = False - - @property - def frequency(self) -> str: - """Frequency of the model.""" - return self.model.frequency - - @property - def module(self) -> str: - """Module of the model.""" - return self.model.source.module - - @property - def ref_path(self) -> Path: - """Path to the reference dataset.""" - return ( - self.model._ref_path - / self.model.source.weather_config - / f"{self.year:04d}" - / f"{self.month:02d}" - ) - - @property - def files(self) -> list[Path]: - """List of files in the model dataset. This could potentially include any - files that are not prepared yet.""" - - atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) - if atomic_files is None: - raise ValueError( - f"Model files for {self.year:04d}-{self.month:02d} not found." - ) - - files = [] - for f in atomic_files: - p = f.path.relative_to(self.ref_path).with_stem(f.path.stem + ".params") - files.append(self.path / p) - return files - - @property - def ref_files(self) -> list[Path]: - """List of files in the reference dataset.""" - - atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) - if atomic_files is None: - raise ValueError( - f"Reference files for {self.year:04d}-{self.month:02d} not found." - ) - - return [f.path for f in atomic_files if f.path.exists()] - - @property - def ref_params(self) -> dict[Path, Path]: - """Reference parameters of the model.""" - - ref_params = {} - for file in self.ref_files: - if file.name.endswith(".params.nc"): - ref_params[file] = self.path / file.name - else: - ref_params[file] = self.path / f"{file.stem}.params.nc" - return ref_params - - @property - def path(self) -> Path: - """Path to the model dataset.""" - return ( - model_dir - / self.module - / self.model.__class__.__name__ - / f"{self.year:04d}" - / f"{self.month:02d}" - ) - - @property - def prepared(self) -> bool: - """Check if the model is prepared. - - Returns: - bool: True if prepared. - """ - if not self._prepared: - self._prepared = self._check_prepared() - return self._prepared - - def _check_prepared(self) -> bool: - """Check if the model is prepared. - - Returns: - bool: True if prepared. - """ - - assert self.path is not None, "The model saving path has not been set yet." - - if not (self.path / "meta.json").exists(): - logger.warning( - "Model %s-%s does not have metadata. Please prepare the model first.", - self.year, - self.month, - ) - return False - - match self.frequency: - case "daily": - with ThreadPoolExecutor( - max_workers=os.getenv("MAX_WORKERS") - ) as executor: - files = [(f, self._hashes.get(f.name)) for f in self.files] - results = list( - tqdm( - executor.map(lambda t: check_hash(*t), files), - total=len(files), - unit="file", - dynamic_ncols=True, - desc=f"Model Files Integrity Check {self.year:04d}-{self.month:02d}", - ) - ) - for file, (is_valid, hash_value) in zip(files, results): - if not is_valid: - logger.warning( - "File %s in model has been modified since model creation. Model is not prepared!", - file, - ) - return False - return True - case "monthly": - return check_hash(self.path / f"{self.month:02d}.params.nc")[0] - case _: - raise ValueError( - f"Frequency {self.frequency} is not supported. Supported frequencies are: daily, monthly." - ) - - def register(self, dataset: xr.Dataset): - """Register the model result with the dataset. - - Args: - dataset (xr.Dataset): Dataset to register. - """ - if not isinstance(dataset, xr.Dataset): - raise ValueError( - f"Dataset must be an xarray Dataset, but got {type(dataset)}." - ) - - match self.frequency: - case "daily": - day = dataset.get("valid_time").dt.day.values[0] - dataset.to_netcdf(self.path / f"{day:02d}.params.nc") - with open(self.path / f"{day:02d}.params.nc", "rb") as f: - self._hashes[f"{day:02d}.params.nc"] = hashlib.sha256( - f.read() - ).hexdigest() - - case "monthly": - dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc") - with open(self.path / f"{self.month:02d}.params.nc", "rb") as f: - self._hashes[f"{self.month:02d}.params.nc"] = hashlib.sha256( - f.read() - ).hexdigest() - - case _: - raise ValueError( - f"Frequency {self.frequency} is not supported. Supported frequencies are: daily, monthly." - ) - - @classmethod - def from_year_month(cls, model: "BaseModel", year: int, month: int) -> Self: - """Create an AtomicModel from year and month. - - Args: - model (BaseModel): BaseModel object. - year (int): Year of the model. - month (int): Month of the model. - - Returns: - AtomicModel: AtomicModel object. - """ - - if not (1 <= month <= 12): - raise ValueError(f"Month {month} is not valid. Must be between 1 and 12.") - if not (2000 <= year <= 2100): - raise ValueError( - f"Year {year} is not valid. Must be between 2000 and 2100." - ) - - path = ( - model_dir - / model.source.module - / model.__class__.__name__ - / f"{year:04d}" - / f"{month:02d}" - / "meta.json" - ) - - if not path.exists(): - return cls(model=model, year=year, month=month) - - with open(path, "r") as f: - data = json.load(f) - - if data["year"] != year or data["month"] != month: - raise ValueError( - f"Model year {data['year']} and month {data['month']} do not match {year} and {month}." - ) - - return cls.from_dict(data, model) - - def __repr__(self): - return f"AtomicModel(year={self.year}, month={self.month}, ref_path={self.ref_path}, path={self.path} {len(self.files)} / {len(self.ref_files)})" - - @classmethod - def from_dict(cls, data: dict, model: "BaseModel") -> Self: - """Create an AtomicModel from a dictionary. - - Args: - data (dict): Dictionary with the model data. - - Returns: - AtomicModel: AtomicModel object. - """ - - inst = cls(year=data["year"], month=data["month"], model=model) - - inst._hashes = data.get("hashes", {}) - inst.prepared - return inst - - def to_dict(self) -> dict: - """Convert the AtomicModel to a dictionary. - - Returns: - dict: Dictionary with the model data. - """ - - return { - "year": self.year, - "month": self.month, - "ref_path": str(self.ref_path), - "path": str(self.path), - "hashes": self._hashes, - "prepared": self.prepared, - } - - def dump(self): - """Dump the model result to a file. - - Returns: - dict: Dictionary with the model data. - """ - info = self.to_dict() - - with open(self.path / "meta.json", "w") as f: - json.dump(info, f, indent=4) - logger.info("Model result dumped to %s", self.path / "meta.json") +from .results import DailyModelResult, MonthlyModelResult, ResultType class BaseModel(abc.ABC): @@ -319,19 +38,8 @@ class BaseModel(abc.ABC): """ SUPPORTED_WEATHER_DATA_CONFIGS: tuple[str] - metadata_keys: set[str] = { - "name", - "module", - "years", - "months", - "files_orig", - "files_prepared", - "weather_data_config", - } def __init__(self, source: BaseDataset, **kwargs): - if not isinstance(source, BaseDataset): - raise ValueError(f"Source must be a Dataset, but got {type(source)}.") if source.weather_config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: raise ValueError( f"Weather data config {source.weather_config} is not supported by this model." @@ -345,7 +53,7 @@ def __init__(self, source: BaseDataset, **kwargs): self._prepared = False self._ref_path = model_dir.parent / self.source.module - self._results = self._prepare_results() + self._results: dict[int, dict[int, ResultType]] = self._prepare_results() def __repr__(self): return f"Model(source={self.source}, type={self.type})" @@ -360,7 +68,7 @@ def frequency(self) -> str: def type(self) -> str: """Type of the model.""" - def _prepare_results(self) -> dict[int, dict[int, ModelResult]]: + def _prepare_results(self) -> dict[int, dict[int, ResultType]]: """Prepare the results of the model. Returns: @@ -370,11 +78,19 @@ def _prepare_results(self) -> dict[int, dict[int, ModelResult]]: years = list(range(self.source.years.start, self.source.years.stop + 1)) months = list(range(self.source.months.start, self.source.months.stop + 1)) - results: dict[int, dict[int, ModelResult]] = {} + results: dict[int, dict[int, ResultType]] = {} for year in years: results[year] = {} for month in months: - results[year][month] = ModelResult.from_year_month(self, year, month) + match self.frequency: + case "daily" | "hourly": + results[year][month] = DailyModelResult.from_year_month( + self, year, month + ) + case "monthly": + results[year][month] = MonthlyModelResult.from_year_month( + self, year, month + ) results[year][month].path.mkdir(parents=True, exist_ok=True) return results @@ -388,8 +104,20 @@ def results(self): """ return self._results + def get_result_year_month(self, years: slice, months: slice) -> list[ResultType]: + """Get the result of the model for a given year and month range. + + Args: + years (slice): Year range. + months (slice): Month range. + Returns: + list: List of DailyModelResult objects. + """ + year_d = [self._results[y] for y in range(years.start, years.stop + 1)] + return [y[m] for y in year_d for m in range(months.start, months.stop + 1)] + @property - def flattened_results(self) -> list[ModelResult]: + def flattened_results(self) -> list[ResultType]: """Flatten the results of the model. Returns: @@ -404,35 +132,42 @@ def flattened_results(self) -> list[ModelResult]: def estimate( self, - height: int, - years: slice, + years: Optional[slice] = None, months: Optional[slice] = None, xs: Optional[slice] = None, ys: Optional[slice] = None, - use_real_data: bool = False, + **kwargs, ) -> xr.DataArray: """Estimate the wind speed at given coordinates. Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. + years (slice, optional): Years. months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. + xs (slice, optional): X coordinates. If None, all x coordinates in source are estimated. + ys (slice, optional): Y coordinates. If None, all y coordinates in source are estimated. + **kwargs: Additional keyword arguments to pass to the model. Returns: xr.DataArray: Dataset with wind speed. """ + if years is None and months is None: + results = self.flattened_results + elif months is None: + results = self.get_result_year_month(years, slice(1, 13)) + else: + results = self.get_result_year_month(years, months) + + files = sum([result.files for result in results], []) + params = xr.open_mfdataset(files) + + if xs is not None: + params = params.sel(x=xs) + if ys is not None: + params = params.sel(y=ys) - return self._estimate_dataset( - height=height, - years=years, - months=months, - xs=xs, - ys=ys, - use_real_data=use_real_data, - ) + output = self._estimate_dataset(params, **kwargs) + params.close() + return output @property def prepared(self) -> bool: @@ -460,22 +195,17 @@ def prepare(self, force: bool = False): force (bool, optional): Force re-prepare the model. Defaults to False. """ - self._prepared = False # NOTE: force re-checking preparedness here if self.prepared and not force: logger.info("The model is already prepared.") return - for result in self.flattened_results: + for result in tqdm(self.flattened_results): if not result.prepared: shutil.rmtree(result.path, ignore_errors=True) result.path.mkdir(parents=True, exist_ok=True) - for ref in tqdm(result.ref_params): - if not ref.exists(): - raise FileNotFoundError(f"Reference file {ref} does not exist.") - ref_ds = xr.open_dataset(ref) - prepared_ds = self._prepare_dataset(ref_ds) - ref_ds.close() + with xr.open_mfdataset(result.ref_files) as ds: + prepared_ds = self._prepare_dataset(ds) result.register(prepared_ds) result.dump() @@ -493,25 +223,13 @@ def _prepare_dataset(self, source: xr.Dataset) -> xr.Dataset: """ @abc.abstractmethod - def _estimate_dataset( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: + def _estimate_dataset(self, params: xr.Dataset, **kwargs) -> xr.DataArray: """Estimate the wind speed from a dataset. Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. - months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. + params (xr.Dataset): Parameters of the model. + **kwargs: Additional keyword arguments to pass to the model. Returns: - xr.DataArray: Dataset with wind speed. + xr.DataArray: Result after modeling. """ diff --git a/src/geodata/model/results/__init__.py b/src/geodata/model/results/__init__.py new file mode 100644 index 00000000..0f3bc4f7 --- /dev/null +++ b/src/geodata/model/results/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""An interface for model computation results.""" + +from .daily import DailyModelResult +from .monthly import MonthlyModelResult + +ResultType = DailyModelResult | MonthlyModelResult + +__all__ = ["DailyModelResult", "MonthlyModelResult", "ResultType"] diff --git a/src/geodata/model/results/_base.py b/src/geodata/model/results/_base.py new file mode 100644 index 00000000..3c906da8 --- /dev/null +++ b/src/geodata/model/results/_base.py @@ -0,0 +1,252 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Self + +import xarray as xr + +from geodata.config import model_dir + +if TYPE_CHECKING: + from .._base import BaseModel + +logger = logging.getLogger(__name__) + + +@dataclass +class BaseModelResult(abc.ABC): + """Model result class. This class is used to store the result of a model. + All results, regardless of the actual storage format, are grouped in monthly + instances. + + Args: + year (int): Year of the model. + month (int): Month of the model. + model (BaseModel): Model object that represent the downloaded raw data. + """ + + year: int + month: int + model: "BaseModel" + + _hashes: dict[str, str] = field(default_factory=dict) + _prepared: bool = False + _frequency: str = field(init=False) + + def __post_init__(self): + if hasattr(self, "_frequency") and self._frequency != self.model.frequency: + raise ValueError( + f"Model frequency {self.model.frequency} does not match {self._frequency}." + ) + self._frequency = self.model.frequency + + @property + def frequency(self) -> str: + """Frequency of the model.""" + return self.model.frequency + + @property + def module(self) -> str: + """Module of the model.""" + return self.model.source.module + + @property + def ref_path(self) -> Path: + """Path to the reference dataset.""" + return ( + self.model._ref_path + / self.model.source.weather_config + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def files(self) -> list[Path]: + """List of files in the model dataset. This could potentially include any + files that are not prepared yet.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Model files for {self.year:04d}-{self.month:02d} not found." + ) + + files = [] + for f in atomic_files: + p = f.path.relative_to(self.ref_path) + files.append(self.path / p) + return files + + @property + def ref_files(self) -> list[Path]: + """List of files in the reference dataset.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Reference files for {self.year:04d}-{self.month:02d} not found." + ) + + return [f.path for f in atomic_files if f.path.exists()] + + @property + def ref_params(self) -> dict[Path, Path]: + """Reference parameters of the model.""" + + ref_params = {} + for file in self.ref_files: + if file.name.endswith(".params.nc"): + ref_params[file] = self.path / file.name + else: + ref_params[file] = self.path / f"{file.stem}.params.nc" + return ref_params + + @property + def path(self) -> Path: + """Path to the model dataset.""" + return ( + model_dir + / self.module + / self.model.__class__.__name__ + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + if not self._prepared: + self._prepared = self._check_prepared() + return self._prepared + + @abc.abstractmethod + def _check_prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + + @abc.abstractmethod + def register(self, dataset: xr.Dataset): + """Register the model result with the dataset. + + Args: + dataset (xr.Dataset): Dataset to register. + """ + + @classmethod + def from_year_month(cls, model: "BaseModel", year: int, month: int) -> Self: + """Create an ModelResult from a year and month. + + Args: + model (BaseModel): BaseModel object. + year (int): Year of the model. + month (int): Month of the model. + + Returns: + AtomicModel: AtomicModel object. + """ + + if not (1 <= month <= 12): + raise ValueError(f"Month {month} is not valid. Must be between 1 and 12.") + if not (2000 <= year <= 2100): + raise ValueError( + f"Year {year} is not valid. Must be between 2000 and 2100." + ) + + path = ( + model_dir + / model.source.module + / model.__class__.__name__ + / f"{year:04d}" + / f"{month:02d}" + / "meta.json" + ) + + if not path.exists(): + return cls(model=model, year=year, month=month) + + with open(path, "r") as f: + data = json.load(f) + + if data["year"] != year or data["month"] != month: + raise ValueError( + f"Model year {data['year']} and month {data['month']} do not match {year} and {month}." + ) + + return cls.from_dict(data, model) + + def __repr__(self): + return f"AtomicModel(year={self.year}, month={self.month}, ref_path={self.ref_path}, path={self.path} {len(self.files)} / {len(self.ref_files)})" + + @classmethod + def from_dict(cls, data: dict, model: "BaseModel") -> Self: + """Create an AtomicModel from a dictionary. + + Args: + data (dict): Dictionary with the model data. + + Returns: + AtomicModel: AtomicModel object. + """ + + inst = cls(year=data["year"], month=data["month"], model=model) + + inst._hashes = data.get("hashes", {}) + inst.prepared + return inst + + def to_dict(self) -> dict: + """Convert the AtomicModel to a dictionary. + + Returns: + dict: Dictionary with the model data. + """ + + return { + "year": self.year, + "month": self.month, + "ref_path": str(self.ref_path), + "path": str(self.path), + "hashes": self._hashes, + "prepared": self.prepared, + } + + def dump(self): + """Dump the model result to a file. + + Returns: + dict: Dictionary with the model data. + """ + info = self.to_dict() + with open(self.path / "meta.json", "w") as f: + json.dump(info, f, indent=4) + + # We dump again to check for meta preparedness + info = self.to_dict() + with open(self.path / "meta.json", "w") as f: + json.dump(info, f, indent=4) + + logger.info("Model result dumped to %s", self.path / "meta.json") diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py new file mode 100644 index 00000000..f04ee793 --- /dev/null +++ b/src/geodata/model/results/daily.py @@ -0,0 +1,119 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import hashlib +import logging +import os +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path + +import xarray as xr +from tqdm.auto import tqdm + +from geodata.utils import check_hash + +from ._base import BaseModelResult + +logger = logging.getLogger(__name__) + + +@dataclass +class DailyModelResult(BaseModelResult): + """Class for daily model results.""" + + def _check_prepared(self): + assert self.path is not None, "The model saving path has not been set yet." + + if not (self.path / "meta.json").exists(): + logger.warning( + "Model %s-%s does not have metadata. Please prepare the model first.", + self.year, + self.month, + ) + return False + + with ThreadPoolExecutor(max_workers=os.getenv("MAX_WORKERS")) as executor: + files = [(f, self._hashes.get(f.name)) for f in self.files] + results = list( + tqdm( + executor.map(lambda t: check_hash(*t), files), + total=len(files), + unit="file", + dynamic_ncols=True, + desc=f"Model Files Integrity Check {self.year:04d}-{self.month:02d}", + ) + ) + + for file, (is_valid, hash_value) in zip(files, results): + if not is_valid: + logger.warning( + "File %s in model has been modified since model creation. Model is not prepared!", + file, + ) + return False + return True + + def register(self, dataset: xr.Dataset): + """Register the model result with the dataset. This file should be a file + covering the entire model period. + + Args: + dataset (xr.Dataset): The dataset to register with. + """ + + time = dataset.get("valid_time") + + start = time.min() + end = time.max() + + if start.dt.month != self.month or end.dt.month != self.month: + raise ValueError( + f"Dataset start month {start.dt.month} does not match model month {self.month}." + ) + + if start.dt.year != self.year or end.dt.year != self.year: + raise ValueError( + f"Dataset start year {start.dt.year} does not match model year {self.year}." + ) + + days, datasets = zip(*dataset.groupby("valid_time.day")) + paths = [self.path / f"{day:02d}.nc" for day in days] + + logger.debug("Saving model results to %s", self.path) + xr.save_mfdataset(datasets, paths) + + # Write the hash file for integrity checking + with ThreadPoolExecutor() as executor: + + def compute_hash(path: Path): + sha256 = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + return path.name, sha256.hexdigest() + + results = list( + tqdm( + executor.map(compute_hash, paths), + total=len(paths), + unit="file", + dynamic_ncols=True, + desc="Computing File Hashes", + ) + ) + + for k, v in results: + self._hashes[k] = v diff --git a/src/geodata/model/results/monthly.py b/src/geodata/model/results/monthly.py new file mode 100644 index 00000000..b8648ffa --- /dev/null +++ b/src/geodata/model/results/monthly.py @@ -0,0 +1,51 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import hashlib +import logging +from dataclasses import dataclass + +import xarray as xr + +from geodata.utils import check_hash + +from ._base import BaseModelResult + +logger = logging.getLogger(__name__) + + +@dataclass +class MonthlyModelResult(BaseModelResult): + """Class for monthly model results.""" + + def _check_prepared(self): + assert self.path is not None, "The model saving path has not been set yet." + + if not (self.path / "meta.json").exists(): + logger.warning( + "Model %s-%s does not have metadata. Please prepare the model first.", + self.year, + self.month, + ) + return False + + return check_hash(self.path / f"{self.month:02d}.params.nc")[0] + + def register(self, dataset: xr.Dataset): + dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc") + with open(self.path / f"{self.month:02d}.params.nc", "rb") as f: + self._hashes[f"{self.month:02d}.params.nc"] = hashlib.sha256( + f.read() + ).hexdigest() diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index 0cd2e7be..eba93d19 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -14,7 +14,7 @@ # along with this program. If not, see . import logging -from typing import Hashable, Optional +from typing import Hashable import numpy as np import scipy.interpolate as sinterp @@ -22,7 +22,7 @@ from xarray.namedarray.pycompat import array_type from ...logging import logger -from ...utils import get_daterange +from ...utils import rechunk_dataset from ._base import WindBaseModel # See https://confluence.ecmwf.int/display/UDOC/L137+model+level+definitions @@ -122,6 +122,24 @@ def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: ) +def _splev_ker(c: np.ndarray, t: np.ndarray, k: int, height: np.ndarray) -> np.ndarray: + return np.atleast_1d(sinterp.splev(height, (t, c, k))) + + +def _splev(da: xr.DataArray, height: float) -> xr.DataArray: + height = np.atleast_1d(height) + return xr.apply_ufunc( + _splev_ker, + da["c"], + input_core_dims=[["height"]], + output_core_dims=[[]], + vectorize=True, + dask="parallelized", + output_dtypes=[da["c"].dtype], + kwargs={"t": da.attrs["t"], "k": da.attrs["k"], "height": height}, + ) + + class WindInterpolationModel(WindBaseModel): """Wind speed estimation based on a spline interpolation of the wind speed at different heights. @@ -171,48 +189,13 @@ def _prepare_dataset( logger.debug("Shape of heights: %s", ds["height"].shape) speeds = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + params = _splrep(speeds, "height") if half_precision: - speeds = speeds.astype(np.float32) + params = params.astype(np.float32) - return _splrep(speeds, "height") + return params - def _estimate_dataset( - self, - height: int, - years: Optional[slice] = None, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.Dataset: - params = xr.open_mfdataset(self.files).transpose("height", ...) - - if not (xs is None or ys is None): - params = params.sel(latitude=ys, longitude=xs) - - if not (years is None and months is None): - if months is None: - months = slice(1, 13) - params = params.sel( - valid_time=get_daterange(years, months), - ) - - # If the height is in the list of known heights, we can directly return - # the wind speed to save computation time. - if float(height) in LEVEL_TO_HEIGHT.values(): - params = params.sel(height=height) - return ( - ((params["u"] ** 2 + params["v"] ** 2) ** 0.5) - .drop("height") - .drop("model_level") - ) - - params = params[["c"]] - spline_params = sinterp.BSpline( - params.attrs.get("t"), params.get("c").values, k=3 - ) - - params = params.drop_dims("height") - return xr.DataArray( - spline_params(height), dims=params.dims, coords=params.coords - ) + def _estimate_dataset(self, params: xr.Dataset, height: float) -> xr.Dataset: + params = params.transpose("height", ...) + params = rechunk_dataset(params, force_full_chunk_dims=["height"]) + return _splev(params, height) diff --git a/src/geodata/utils.py b/src/geodata/utils.py index 951d996a..b8aae82f 100644 --- a/src/geodata/utils.py +++ b/src/geodata/utils.py @@ -18,8 +18,10 @@ import json from pathlib import Path +import dask.array as da import numpy as np import pandas as pd +import xarray as xr def dummy_njit(f=None, *args, **kwargs): @@ -115,3 +117,66 @@ def check_hash(file: Path, saved_hash: str | None = None) -> tuple[bool, str]: return True, computed_hash else: return computed_hash == saved_hash, computed_hash + + +def rechunk_dataset( + data: xr.DataArray | xr.Dataset, + target_chunk_bytes: int = 20 * 1024**2, + force_full_chunk_dims: list[str] | None = None, +): + """ + Rechunk xarray DataArray or Dataset to maximize chunk size + under a memory limit, while forcing certain dimensions to be unchunked + (i.e., use only one chunk across that dimension). + + Parameters: + data: xr.DataArray or xr.Dataset + target_chunk_bytes: maximum memory per chunk (in bytes) + force_full_chunk_dims: list of dimension names to not chunk (single chunk along that dim) + """ + if force_full_chunk_dims is None: + force_full_chunk_dims = [] + + if isinstance(data, xr.Dataset): + vars_to_chunk = { + name: rechunk_dataset(var, target_chunk_bytes, force_full_chunk_dims) + for name, var in data.data_vars.items() + } + return data.assign(vars_to_chunk) + + if not isinstance(data.data, da.Array): + raise ValueError("Data must be a Dask-backed xarray object") + + shape = data.shape + dims = data.dims + itemsize = data.dtype.itemsize + + # Start with full dims + chunk_shape = list(shape) + dim_to_index = {dim: i for i, dim in enumerate(dims)} + + # Force full chunks on specified dims + for dim in force_full_chunk_dims: + if dim in dim_to_index: + chunk_shape[dim_to_index[dim]] = shape[dim_to_index[dim]] + + # Reduce non-fixed dims to fit memory budget + while True: + est_bytes = np.prod(chunk_shape) * itemsize + if est_bytes <= target_chunk_bytes: + break + + # Pick largest non-fixed dimension to halve + candidates = [ + (i, size) + for i, size in enumerate(chunk_shape) + if dims[i] not in force_full_chunk_dims and size > 1 + ] + if not candidates: + break # Can't reduce further + + i, _ = max(candidates, key=lambda x: x[1]) + chunk_shape[i] = max(1, chunk_shape[i] // 2) + + chunk_dict = dict(zip(dims, chunk_shape)) + return data.chunk(chunk_dict) From ccf4fae6efa5b4a82bdeef0b2605c146b663469f Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 12 May 2025 12:59:30 -0700 Subject: [PATCH 38/54] feat: include h5netcdf for better performance --- pyproject.toml | 1 + src/geodata/model/_base.py | 21 ++++++++++++++--- uv.lock | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d7532949..4b35ec84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pyyaml>=6.0.2", "dask>=2024.9.0", "tqdm>=4.66.5", + "h5netcdf>=1.6.1", ] requires-python = ">=3.10" readme = "README.md" diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 1decabb1..473a9746 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -26,6 +26,19 @@ from ..logging import logger from .results import DailyModelResult, MonthlyModelResult, ResultType +try: + import h5netcdf + + XR_PARALLEL = True + XR_ENGINE = "h5netcdf" +except ImportError: + XR_PARALLEL = False + XR_ENGINE = None + logger.warning( + "h5netcdf is not installed. Parallel reading of netCDF files will be disabled. " + "This could have some performance implications." + ) + class BaseModel(abc.ABC): """Base class for geospatial modeling. @@ -158,7 +171,7 @@ def estimate( results = self.get_result_year_month(years, months) files = sum([result.files for result in results], []) - params = xr.open_mfdataset(files) + params = xr.open_mfdataset(files, engine=XR_ENGINE, parallel=XR_PARALLEL) if xs is not None: params = params.sel(x=xs) @@ -200,11 +213,13 @@ def prepare(self, force: bool = False): return for result in tqdm(self.flattened_results): - if not result.prepared: + if not result.prepared or force: shutil.rmtree(result.path, ignore_errors=True) result.path.mkdir(parents=True, exist_ok=True) - with xr.open_mfdataset(result.ref_files) as ds: + with xr.open_mfdataset( + result.ref_files, engine=XR_ENGINE, parallel=XR_PARALLEL + ) as ds: prepared_ds = self._prepare_dataset(ds) result.register(prepared_ds) diff --git a/uv.lock b/uv.lock index 66a8ec76..6b6aeb7b 100644 --- a/uv.lock +++ b/uv.lock @@ -799,6 +799,7 @@ dependencies = [ { name = "bottleneck" }, { name = "dask" }, { name = "geopandas" }, + { name = "h5netcdf" }, { name = "matplotlib" }, { name = "netcdf4" }, { name = "numexpr" }, @@ -849,6 +850,7 @@ requires-dist = [ { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.5" }, { name = "dask", specifier = ">=2024.9.0" }, { name = "geopandas", specifier = ">=1.0.1" }, + { name = "h5netcdf", specifier = ">=1.6.1" }, { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2024.8.0" }, { name = "matplotlib", specifier = "==3.9.2" }, { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, @@ -959,6 +961,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, ] +[[package]] +name = "h5netcdf" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h5py" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/6d/c1b8e48fccbb588c23033bf219a3190a50813857d78a4c1aae2e1f3969e9/h5netcdf-1.6.1.tar.gz", hash = "sha256:7ef4ecd811374d94d29ac5e7f7db71ff59b55ef8eeefbe4ccc2c316853d31894", size = 64456 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/fc/e73747f3dd31906bfbb78c76069f67d91525fefa28492a1f949cbb4a3c7f/h5netcdf-1.6.1-py3-none-any.whl", hash = "sha256:1ec75cabd6ab50c6e7109d0c6595eb2960ba0e79fef2257607ab80838d84e6f6", size = 49561 }, +] + +[[package]] +name = "h5py" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/2e/a22d6a8bfa6f8be33e7febd985680fba531562795f0a9077ed1eb047bfb0/h5py-3.13.0.tar.gz", hash = "sha256:1870e46518720023da85d0895a1960ff2ce398c5671eac3b1a41ec696b7105c3", size = 414876 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/8a/bc76588ff1a254e939ce48f30655a8f79fac614ca8bd1eda1a79fa276671/h5py-3.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5540daee2b236d9569c950b417f13fd112d51d78b4c43012de05774908dff3f5", size = 3413286 }, + { url = "https://files.pythonhosted.org/packages/19/bd/9f249ecc6c517b2796330b0aab7d2351a108fdbd00d4bb847c0877b5533e/h5py-3.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:10894c55d46df502d82a7a4ed38f9c3fdbcb93efb42e25d275193e093071fade", size = 2915673 }, + { url = "https://files.pythonhosted.org/packages/72/71/0dd079208d7d3c3988cebc0776c2de58b4d51d8eeb6eab871330133dfee6/h5py-3.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb267ce4b83f9c42560e9ff4d30f60f7ae492eacf9c7ede849edf8c1b860e16b", size = 4283822 }, + { url = "https://files.pythonhosted.org/packages/d8/fa/0b6a59a1043c53d5d287effa02303bd248905ee82b25143c7caad8b340ad/h5py-3.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2cf6a231a07c14acd504a945a6e9ec115e0007f675bde5e0de30a4dc8d86a31", size = 4548100 }, + { url = "https://files.pythonhosted.org/packages/12/42/ad555a7ff7836c943fe97009405566dc77bcd2a17816227c10bd067a3ee1/h5py-3.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:851ae3a8563d87a5a0dc49c2e2529c75b8842582ccaefbf84297d2cfceeacd61", size = 2950547 }, + { url = "https://files.pythonhosted.org/packages/86/2b/50b15fdefb577d073b49699e6ea6a0a77a3a1016c2b67e2149fc50124a10/h5py-3.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8a8e38ef4ceb969f832cc230c0cf808c613cc47e31e768fd7b1106c55afa1cb8", size = 3422922 }, + { url = "https://files.pythonhosted.org/packages/94/59/36d87a559cab9c59b59088d52e86008d27a9602ce3afc9d3b51823014bf3/h5py-3.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f35640e81b03c02a88b8bf99fb6a9d3023cc52f7c627694db2f379e0028f2868", size = 2921619 }, + { url = "https://files.pythonhosted.org/packages/37/ef/6f80b19682c0b0835bbee7b253bec9c16af9004f2fd6427b1dd858100273/h5py-3.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:337af114616f3656da0c83b68fcf53ecd9ce9989a700b0883a6e7c483c3235d4", size = 4259366 }, + { url = "https://files.pythonhosted.org/packages/03/71/c99f662d4832c8835453cf3476f95daa28372023bda4aa1fca9e97c24f09/h5py-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:782ff0ac39f455f21fd1c8ebc007328f65f43d56718a89327eec76677ebf238a", size = 4509058 }, + { url = "https://files.pythonhosted.org/packages/56/89/e3ff23e07131ff73a72a349be9639e4de84e163af89c1c218b939459a98a/h5py-3.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:22ffe2a25770a2d67213a1b94f58006c14dce06933a42d2aaa0318c5868d1508", size = 2966428 }, + { url = "https://files.pythonhosted.org/packages/d8/20/438f6366ba4ded80eadb38f8927f5e2cd6d2e087179552f20ae3dbcd5d5b/h5py-3.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:477c58307b6b9a2509c59c57811afb9f598aedede24a67da808262dfa0ee37b4", size = 3384442 }, + { url = "https://files.pythonhosted.org/packages/10/13/cc1cb7231399617d9951233eb12fddd396ff5d4f7f057ee5d2b1ca0ee7e7/h5py-3.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:57c4c74f627c616f02b7aec608a8c706fe08cb5b0ba7c08555a4eb1dde20805a", size = 2917567 }, + { url = "https://files.pythonhosted.org/packages/9e/d9/aed99e1c858dc698489f916eeb7c07513bc864885d28ab3689d572ba0ea0/h5py-3.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:357e6dc20b101a805ccfd0024731fbaf6e8718c18c09baf3b5e4e9d198d13fca", size = 4669544 }, + { url = "https://files.pythonhosted.org/packages/a7/da/3c137006ff5f0433f0fb076b1ebe4a7bf7b5ee1e8811b5486af98b500dd5/h5py-3.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6f13f9b5ce549448c01e4dfe08ea8d1772e6078799af2c1c8d09e941230a90d", size = 4932139 }, + { url = "https://files.pythonhosted.org/packages/25/61/d897952629cae131c19d4c41b2521e7dd6382f2d7177c87615c2e6dced1a/h5py-3.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:21daf38171753899b5905f3d82c99b0b1ec2cbbe282a037cad431feb620e62ec", size = 2954179 }, + { url = "https://files.pythonhosted.org/packages/60/43/f276f27921919a9144074320ce4ca40882fc67b3cfee81c3f5c7df083e97/h5py-3.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e520ec76de00943dd017c8ea3f354fa1d2f542eac994811943a8faedf2a7d5cb", size = 3358040 }, + { url = "https://files.pythonhosted.org/packages/1b/86/ad4a4cf781b08d4572be8bbdd8f108bb97b266a14835c640dc43dafc0729/h5py-3.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e79d8368cd9295045956bfb436656bea3f915beaa11d342e9f79f129f5178763", size = 2892766 }, + { url = "https://files.pythonhosted.org/packages/69/84/4c6367d6b58deaf0fa84999ec819e7578eee96cea6cbd613640d0625ed5e/h5py-3.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56dd172d862e850823c4af02dc4ddbc308f042b85472ffdaca67f1598dff4a57", size = 4664255 }, + { url = "https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be949b46b7388074c5acae017fbbe3e5ba303fd9daaa52157fdfef30bbdacadd", size = 4927580 }, + { url = "https://files.pythonhosted.org/packages/97/34/165b87ea55184770a0c1fcdb7e017199974ad2e271451fd045cfe35f3add/h5py-3.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:4f97ecde7ac6513b21cd95efdfc38dc6d19f96f6ca6f2a30550e94e551458e0a", size = 2940890 }, +] + [[package]] name = "herbie-data" version = "2024.8.0" From c58d5e638de709f0ead6df2a659483f7a08525ac Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 12 May 2025 14:24:46 -0700 Subject: [PATCH 39/54] feat: incorporate parallel registering of model result --- src/geodata/model/results/_base.py | 9 ++++++--- src/geodata/model/results/daily.py | 10 +++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/geodata/model/results/_base.py b/src/geodata/model/results/_base.py index 3c906da8..95cb0f31 100644 --- a/src/geodata/model/results/_base.py +++ b/src/geodata/model/results/_base.py @@ -13,6 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +from __future__ import annotations + import abc import json import logging @@ -44,7 +47,7 @@ class BaseModelResult(abc.ABC): year: int month: int - model: "BaseModel" + model: BaseModel _hashes: dict[str, str] = field(default_factory=dict) _prepared: bool = False @@ -157,7 +160,7 @@ def register(self, dataset: xr.Dataset): """ @classmethod - def from_year_month(cls, model: "BaseModel", year: int, month: int) -> Self: + def from_year_month(cls, model: BaseModel, year: int, month: int) -> Self: """Create an ModelResult from a year and month. Args: @@ -202,7 +205,7 @@ def __repr__(self): return f"AtomicModel(year={self.year}, month={self.month}, ref_path={self.ref_path}, path={self.path} {len(self.files)} / {len(self.ref_files)})" @classmethod - def from_dict(cls, data: dict, model: "BaseModel") -> Self: + def from_dict(cls, data: dict, model: BaseModel) -> Self: """Create an AtomicModel from a dictionary. Args: diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py index f04ee793..5672c7a1 100644 --- a/src/geodata/model/results/daily.py +++ b/src/geodata/model/results/daily.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + import hashlib import logging import os @@ -29,6 +30,13 @@ logger = logging.getLogger(__name__) +try: + import h5netcdf + + XR_ENGINE = "h5netcdf" +except ImportError: + XR_ENGINE = None + @dataclass class DailyModelResult(BaseModelResult): @@ -93,7 +101,7 @@ def register(self, dataset: xr.Dataset): paths = [self.path / f"{day:02d}.nc" for day in days] logger.debug("Saving model results to %s", self.path) - xr.save_mfdataset(datasets, paths) + xr.save_mfdataset(datasets, paths, engine=XR_ENGINE) # Write the hash file for integrity checking with ThreadPoolExecutor() as executor: From 2d53e9a553e9354be58104394fdbbc59e971fc9d Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 16 May 2025 15:35:47 -0700 Subject: [PATCH 40/54] fix: few improvements 1. support quick check of model results 2. prevent estimation when model is not prepared 3. proper max_workers parsing --- src/geodata/model/_base.py | 26 +++++++++++++++++++---- src/geodata/model/results/daily.py | 33 +++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 473a9746..a8a89bd7 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -15,6 +15,8 @@ import abc +import importlib.util +import os import shutil from typing import Optional @@ -26,12 +28,10 @@ from ..logging import logger from .results import DailyModelResult, MonthlyModelResult, ResultType -try: - import h5netcdf - +if importlib.util.find_spec("h5netcdf") is not None: XR_PARALLEL = True XR_ENGINE = "h5netcdf" -except ImportError: +else: XR_PARALLEL = False XR_ENGINE = None logger.warning( @@ -39,6 +39,17 @@ "This could have some performance implications." ) +# Parse the MAX_WORKERS environment variable if present +MAX_WORKERS = os.getenv("MAX_WORKERS") +if MAX_WORKERS is not None: + try: + max_workers = int(MAX_WORKERS) + except ValueError: + logger.warning( + "MAX_WORKERS environment variable is not an integer. Using default value." + ) + MAX_WORKERS = None + class BaseModel(abc.ABC): """Base class for geospatial modeling. @@ -47,6 +58,7 @@ class BaseModel(abc.ABC): name (str): The name of the model. source (BaseDataset): The source of the model. interpolate (bool, optional): Interpolate the source to the same grid as the target. Defaults to False. + quick_check (bool, optional): Quick check for the model. Defaults to False. If True, the model parameters will be checked for presence, but not the integrity. **kwargs: Additional keyword arguments to pass to the model. """ @@ -62,6 +74,7 @@ def __init__(self, source: BaseDataset, **kwargs): raise ValueError("The source Dataset for this model is not prepared.") self.source = source + self.quick_check = kwargs.get("quick_check", False) self._extra_kwargs = kwargs self._prepared = False @@ -163,6 +176,11 @@ def estimate( Returns: xr.DataArray: Dataset with wind speed. """ + if not self.prepared: + raise RuntimeError( + "The model is not prepared. Please prepare the model first." + ) + if years is None and months is None: results = self.flattened_results elif months is None: diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py index 5672c7a1..add67dde 100644 --- a/src/geodata/model/results/daily.py +++ b/src/geodata/model/results/daily.py @@ -16,7 +16,6 @@ import hashlib import logging -import os from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path @@ -26,17 +25,11 @@ from geodata.utils import check_hash +from .._base import MAX_WORKERS, XR_ENGINE from ._base import BaseModelResult logger = logging.getLogger(__name__) -try: - import h5netcdf - - XR_ENGINE = "h5netcdf" -except ImportError: - XR_ENGINE = None - @dataclass class DailyModelResult(BaseModelResult): @@ -53,7 +46,26 @@ def _check_prepared(self): ) return False - with ThreadPoolExecutor(max_workers=os.getenv("MAX_WORKERS")) as executor: + if self.model.quick_check: + # If the quick check is enabled, we only need to check the file hashes + # and not the actual data. + logger.debug( + "Quick check is enabled. Only checking file hashes for model %s-%s.", + self.year, + self.month, + ) + + for file in self.files: + if not file.exists(): + logger.warning( + "File %s in model does not exist. Model is not prepared!", + str(file), + ) + return False + + return True + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: files = [(f, self._hashes.get(f.name)) for f in self.files] results = list( tqdm( @@ -69,9 +81,10 @@ def _check_prepared(self): if not is_valid: logger.warning( "File %s in model has been modified since model creation. Model is not prepared!", - file, + str(file[0]), ) return False + return True def register(self, dataset: xr.Dataset): From 63d9ebf5249dccb07f84bc43861d067977642186 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Thu, 22 May 2025 18:06:12 -0700 Subject: [PATCH 41/54] feat: support CF computation in wind modeling --- src/geodata/model/results/daily.py | 5 ++- src/geodata/model/wind/_base.py | 66 ++++++++++++++++++++++++++++++ src/geodata/resource.py | 26 ++++++++++-- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py index add67dde..1a60d0cf 100644 --- a/src/geodata/model/results/daily.py +++ b/src/geodata/model/results/daily.py @@ -25,7 +25,6 @@ from geodata.utils import check_hash -from .._base import MAX_WORKERS, XR_ENGINE from ._base import BaseModelResult logger = logging.getLogger(__name__) @@ -64,6 +63,7 @@ def _check_prepared(self): return False return True + from .._base import MAX_WORKERS with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: files = [(f, self._hashes.get(f.name)) for f in self.files] @@ -114,6 +114,9 @@ def register(self, dataset: xr.Dataset): paths = [self.path / f"{day:02d}.nc" for day in days] logger.debug("Saving model results to %s", self.path) + + from .._base import XR_ENGINE + xr.save_mfdataset(datasets, paths, engine=XR_ENGINE) # Write the hash file for integrity checking diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index 849c8d51..de7920c4 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -39,8 +39,13 @@ >>> model.estimate(xs=slice(1, 2), ys=slice(1, 2), years=slice(2010, 2010), months=slice(1, 2)) """ +import xarray as xr + +from ...resource import get_windturbineconfig from .._base import BaseModel +from scipy.interpolate import interp1d + HEIGHTS = {"u50m": 50, "u10m": 10, "u2m": 2} @@ -54,3 +59,64 @@ class WindBaseModel(BaseModel): """ type: str = "wind" + + def estimate_power( + self, + turbine: str, + xs: slice | None = None, + ys: slice | None = None, + years: slice | None = None, + months: slice | None = None, + include_raw_power: bool = False, + ) -> None: + """Estimate wind speed at the given locations and times. + + Args: + turbine (str): Turbine name. + xs (slice, optional): X slice. Defaults to None. + ys (slice, optional): Y slice. Defaults to None. + years (slice, optional): Year slice. Defaults to None. + months (slice, optional): Month slice. Defaults to None. + include_raw_power (bool, optional): Include raw power output. Defaults to False. + + Returns: + xr.DataArray: Estimated wind speed. + """ + + # Get the wind turbine configuration + try: + turbineconf = get_windturbineconfig(turbine) + except FileNotFoundError: + raise ValueError(f"Wind turbine configuration '{turbine}' not found.") + + speed = self.estimate( + years=years, months=months, xs=xs, ys=ys, height=turbineconf["hub_height"] + ) + + interp_fn = interp1d( + turbineconf["V"], + turbineconf["POW"], + bounds_error=False, + fill_value="extrapolate", + ) + + # Calculate the power output + power = xr.apply_ufunc( + interp_fn, + speed, + vectorize=True, + dask="parallelized", + output_dtypes=[float], + ) + + if include_raw_power: + # Calculate the capacity factor + cf: xr.DataArray = power / turbineconf["P"] + cf.attrs["units"] = "dimensionless" + cf.attrs["long_name"] = "Capacity factor" + cf.attrs["description"] = "Capacity factor of the wind turbine" + cf.attrs["turbine"] = turbine + + return xr.Dataset({"power": power, "cf": cf}) + + return xr.Dataset({"cf": power / turbineconf["P"]}) diff --git a/src/geodata/resource.py b/src/geodata/resource.py index 7bbcd962..28a9847d 100644 --- a/src/geodata/resource.py +++ b/src/geodata/resource.py @@ -32,14 +32,34 @@ logger = logging.getLogger(name=__name__) -def get_windturbineconfig(turbine): - """Load the 'turbine'.yaml file from local disk and provide a turbine dict.""" +def get_windturbineconfig(turbine: str): + """Load the 'turbine'.yaml file from local disk and provide a turbine dict. + + Args: + turbine (str): Name of the wind turbine configuration file (without .yaml). + + Returns: + dict: Dictionary containing the wind turbine configuration. + - V: Wind speed array + - POW: Power output array + - hub_height: Hub height + - P: Rated power output + """ res_name = "resources/windturbine/" + turbine + ".yaml" with open(SRC_ROOT / res_name, "r") as resource_file: turbineconf = yaml.safe_load(resource_file) V, POW, hub_height = itemgetter("V", "POW", "HUB_HEIGHT")(turbineconf) - return dict(V=np.array(V), POW=np.array(POW), hub_height=hub_height, P=np.max(POW)) + + # Let's make sure that the wind speed and power output arrays are sorted + V = np.asarray(V) + POW = np.asarray(POW) + + sorted_indices = np.argsort(V) + V = V[sorted_indices] + POW = POW[sorted_indices] + + return {"V": V, "POW": POW, "hub_height": hub_height, "P": POW.max()} def get_solarpanelconfig(panel): From d3e552ab6f42b217e5a1ae5d90673fd7c5f41292 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 26 May 2025 17:05:05 -0700 Subject: [PATCH 42/54] feat: pvlib support - adding 2m dew point temperature for relative humidity calculation --- pyproject.toml | 2 + .../datasets/era5/hourly/wind_solar.py | 2 + .../datasets/era5/monthly/wind_solar.py | 1 + uv.lock | 74 +++++++++++++++++++ 4 files changed, 79 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4b35ec84..a5243667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,8 @@ dependencies = [ "dask>=2024.9.0", "tqdm>=4.66.5", "h5netcdf>=1.6.1", + "pvlib>=0.12.0", + "timezonefinder>=6.5.9", ] requires-python = ">=3.10" readme = "README.md" diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py index d376f93f..9ca2da1c 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -43,6 +43,7 @@ class ERA5WindSolarHourlyDataset(ERA5BaseDataset): - 100m_u_component_of_wind - 100m_v_component_of_wind - 2m_temperature + - 2m_dew_point_temperature - runoff - soil_temperature_level_4 - surface_net_solar_radiation @@ -61,6 +62,7 @@ class ERA5WindSolarHourlyDataset(ERA5BaseDataset): "100m_u_component_of_wind": "u100", "100m_v_component_of_wind": "v100", "2m_temperature": "t2m", + "2m_dew_point_temperature": "d2m", "runoff": "ro", "soil_temperature_level_4": "stl4", "surface_net_solar_radiation": "ssr", diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py index bcb3ae5a..8a523b48 100644 --- a/src/geodata/datasets/era5/monthly/wind_solar.py +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -43,6 +43,7 @@ class ERA5WindSolarMonthlyDataset(ERA5WindSolarHourlyDataset): - 100m_u_component_of_wind - 100m_v_component_of_wind - 2m_temperature + - 2m_dew_point_temperature - runoff - soil_temperature_level_4 - surface_net_solar_radiation diff --git a/uv.lock b/uv.lock index 6b6aeb7b..9cd0bc17 100644 --- a/uv.lock +++ b/uv.lock @@ -805,6 +805,7 @@ dependencies = [ { name = "numexpr" }, { name = "numpy" }, { name = "pandas" }, + { name = "pvlib" }, { name = "pyproj" }, { name = "pyyaml" }, { name = "rasterio" }, @@ -812,6 +813,7 @@ dependencies = [ { name = "rioxarray" }, { name = "scipy" }, { name = "shapely" }, + { name = "timezonefinder" }, { name = "toolz" }, { name = "tqdm" }, { name = "xarray" }, @@ -860,6 +862,7 @@ requires-dist = [ { name = "numexpr", specifier = "==2.10.1" }, { name = "numpy", specifier = "<2" }, { name = "pandas", specifier = ">=2.2.3" }, + { name = "pvlib", specifier = ">=0.12.0" }, { name = "pyproj", specifier = "==3.6.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rasterio", specifier = "==1.4.0" }, @@ -870,6 +873,7 @@ requires-dist = [ { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0.0" }, { name = "sphinx-autoapi", marker = "extra == 'docs'", specifier = "==3.3.2" }, { name = "sphinx-book-theme", marker = "extra == 'docs'", specifier = ">=1.1.3" }, + { name = "timezonefinder", specifier = ">=6.5.9" }, { name = "toolz", specifier = ">=0.12.1" }, { name = "tqdm", specifier = ">=4.66.5" }, { name = "xarray", specifier = ">=2024.9.0" }, @@ -961,6 +965,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, ] +[[package]] +name = "h3" +version = "4.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/2a/dac23698122456ce12345e0a1408c23bc389eaffab5eb7689b8e080d9dfb/h3-4.2.2.tar.gz", hash = "sha256:5cc78d546b5c732f480a842e3e436c393fe37fe59b063fe9cb5589206b7c4c7e", size = 167640 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/d8/3a8900058cd463f65a4aa4482ad46f1e813836b18df2bfb15d93aa6dc6d5/h3-4.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:75bae45428b133c3006a3c72646e42856c8800a74e47a818e1bcd242a86e9dae", size = 887980 }, + { url = "https://files.pythonhosted.org/packages/a3/7d/0f50c45edbe36657665a00ef4e5d210d45716f63b360497f4f8677035019/h3-4.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9992eb522636cbcf7c5b5fd32436395bee9423805ee06fd8ad6eca83f4af654", size = 832701 }, + { url = "https://files.pythonhosted.org/packages/0f/6e/6a140e6fb089eb302bf98cf708eb67cea67ffb128ee17b1b11c1cbad4734/h3-4.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7687de29510ece132e59739bd87dcb3fe25783150502685d7d5be266ea4162b5", size = 977189 }, + { url = "https://files.pythonhosted.org/packages/be/1d/84bd08e64d56deeace7de69e24a115faaf2d8c58d0f02d3661258db0369e/h3-4.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51d715d0471bb581bd94f72529c3194e9b91ce30202672d2050161292f774742", size = 1030504 }, + { url = "https://files.pythonhosted.org/packages/e6/61/602ec16ef539f7285a6fab99beb2f15903f8598e3659d478003979b7ce7a/h3-4.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f4eab1fdfc15b83edac415377790114cae5f117ae7d3293fc9b27354fb2987e6", size = 1078345 }, + { url = "https://files.pythonhosted.org/packages/85/22/5bd067d062fdd43c1093fbe8dbd869c9e6cc80277501a25461c0459cbf51/h3-4.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:6c2677a20c46148a31838e5b4eed11554fd83a9f07c63a2056f4187cb1ba605f", size = 807675 }, + { url = "https://files.pythonhosted.org/packages/af/b7/ef768dad3f93ccb33823c8e9ef3cab3fc23f863f5536ac93e3462d9d79ea/h3-4.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b4039d3bc8236ea371402c755867c6b0d36d072dcd1117e22a9040dd63ba522c", size = 888300 }, + { url = "https://files.pythonhosted.org/packages/cb/51/8aba727a3b453ae62b5a47cfba8cb4dc1c29f209375ed694ef0fe030c855/h3-4.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d637189edf2f1523d625e03415ce4b7fdfe224e6254b73c987b8774214a09edf", size = 832835 }, + { url = "https://files.pythonhosted.org/packages/01/e0/2b2b484416d047f932b3d01cc84f0e8b187abece5ab9044a25c1fa76bfb6/h3-4.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5b88187fcfc88a4315633b0cf27fa55b736948b78833045d03a08a33db4feb3", size = 977615 }, + { url = "https://files.pythonhosted.org/packages/ff/c0/f7986dce65bfdf6812a2fe85aa73f636a9c1ce547cdc481b7b7110abbfba/h3-4.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2459e482147a37292e9638f08bf3b4f32df972457064721b888883c1883848c9", size = 1031287 }, + { url = "https://files.pythonhosted.org/packages/7d/03/d4519bde23b1457fe32d4063e3cc8b95fa61b6abd4c322dcdaafe47421be/h3-4.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4db576b1d96d3d828f1b06eef943e1ad00eb9ff2caae5fb894ec3f4badcfde34", size = 1078669 }, + { url = "https://files.pythonhosted.org/packages/eb/0d/00d8d9b688793f09976cae6e731e7c32641af9d010dec79410f5da13db5c/h3-4.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:db1dc702760a8b94abbd1ddb6efd1e88c63032f0bbd7288159af54ce6a611f3a", size = 808885 }, + { url = "https://files.pythonhosted.org/packages/57/29/853abc62ca5526163b29ec8be0b1229f87d99ac7e7565be738856eda3091/h3-4.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c09b7df785cf8a191e33e3b15ba34aef864b78061b2aeaee87a434aa703bb29c", size = 899353 }, + { url = "https://files.pythonhosted.org/packages/e3/2e/0ade7631f0bb959c72c6eaab05f4d4f009224d0062950975dbf7d5f0db38/h3-4.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:35afe80a011abf9ffa8bc885b847a01a0909fca6556f6f997d6adf7beec87e38", size = 837002 }, + { url = "https://files.pythonhosted.org/packages/33/e9/b75c1cdc11b68064e82720f66d22a97e8296aa59194d132b6cbf09c9fd34/h3-4.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a56aa550e56de5fc9726fd20f484b23a336d6b0844e40604500ce9c3bc8f86c", size = 959920 }, + { url = "https://files.pythonhosted.org/packages/fd/b0/2fb42bd02f5d11baba3e81ed524ec1c4ab92ca73050316f2d2955ecf9c1d/h3-4.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e83d7484441f42a6bcd0ffc9626387e4ff8a7563ae659667f5b59b814fe198b7", size = 1014727 }, + { url = "https://files.pythonhosted.org/packages/5c/10/4a29b577f27e94f087733f3ebffa918e1e66ef22e0d1862ef52f4c9aa040/h3-4.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:737d7cf3aeebc60b0e091220a1bdf94608f16457589cf67acc288444ee5f5dc8", size = 1061560 }, + { url = "https://files.pythonhosted.org/packages/25/15/717815d4b34d2776241de2815ab415aa86a097abf6c76c6b5b67f5954927/h3-4.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bc61382bceff776f0d6278e6ba54d94c9e15137d527adbd50914c7591f9c4460", size = 813445 }, + { url = "https://files.pythonhosted.org/packages/4f/bb/e362d88633f309633f28d958ec0eb813166711972d4538afd337d151169e/h3-4.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d70059de58956d58a92213a0b7adbc4fcf643a102911bc1ad637c99554765add", size = 891287 }, + { url = "https://files.pythonhosted.org/packages/a7/99/0827284f7a8dfe4b7680d5c2143a16d49fa2d2e01a26e6af08b992d96003/h3-4.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a9b32da65d9ed6f97ca9ff404721c97f0faec5656e205a7601628ae402f2b282", size = 829418 }, + { url = "https://files.pythonhosted.org/packages/0d/33/67ffd531b1b9b7f4b01da7f7fcad29b367909146bf3ebc290c3267c02580/h3-4.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bea5b734343c5074c60a575cb609157f51aa1c8da5c4064491bb5c58a4dac55", size = 954805 }, + { url = "https://files.pythonhosted.org/packages/0d/1b/514cfd2dffdf18259b39dc0178a10cb15257ef5365bcbb99b2c682f96c89/h3-4.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7836e8e32c30f34fba1c26cdce8e4e7b4a9f48d18f3a2ab9edfeee995deb92a", size = 1008989 }, + { url = "https://files.pythonhosted.org/packages/59/5d/db71fc5e8d82aa63717d3c12ad9e482b128f66a76f11c17f9c80701f12a8/h3-4.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:652299d3bef76af65f5166bc4af0598afa6c7d93cfc13c7a645a95d8c8e5b10e", size = 1058255 }, + { url = "https://files.pythonhosted.org/packages/b8/c5/6088ff388e73376fc672752059df37bf17b13b3f3b8ed55d178f2833e89c/h3-4.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfda0f2864462f074cc22df75a50b9db010988e341a1871993675bb21954891b", size = 810071 }, +] + [[package]] name = "h5netcdf" version = "1.6.1" @@ -2202,6 +2238,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "pvlib" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h5py" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pytz" }, + { name = "requests" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/54/d7b2f3ee461fecd1ef5b2d24aa32d2e167f260c39c614dbd8cc4285937ce/pvlib-0.12.0.tar.gz", hash = "sha256:222bcb7b88f228944e7f7e9317082b7182232b8d0a5d1b1499cff50027d1e964", size = 35715594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/2b/c3766d6a78ab60ff1d300a1d3479134bdae6073c84bd310fc95912b7516e/pvlib-0.12.0-py3-none-any.whl", hash = "sha256:5922991b6a1ad9c4c90b114b582c8ac91ec4bb2d5fb04ce00eac83581316fa2e", size = 19325114 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -3061,6 +3114,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154 }, ] +[[package]] +name = "timezonefinder" +version = "6.5.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "h3" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/4d/8694391d6b014bc49f8fd2eb8c05c94526b36fba8dc76d439f3d51948e46/timezonefinder-6.5.9.tar.gz", hash = "sha256:0d84c792a499fd098a35c701c3e3293423ba8d45c81b3eecd7c7cb72c7f1f415", size = 51435008 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/d0/e24bdd01bfda4f66fac30e58a339d1d733718000b517c93ebfa792f946c0/timezonefinder-6.5.9-cp310-cp310-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52d5a4a8fc96990f72d9d3d48297e789217f67689d3178c9ff8ea3ab57125e3b", size = 51443226 }, + { url = "https://files.pythonhosted.org/packages/68/6a/321f849a41ac598981dfded6b9e2c0741b0406e6b5aa54f475058f156af4/timezonefinder-6.5.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a84ad5afb02ca1b536481cee05a8f2d5d7dd4818f73cd780acd03aa3cc033c9", size = 51445162 }, + { url = "https://files.pythonhosted.org/packages/0e/24/6b10ae9aae25871d6776c78b77322632b023f4d5a49d9c657f268a2ec568/timezonefinder-6.5.9-cp311-cp311-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:643535f76436b13216ed1c3b69c6ed8f793253810916ca6bef3b8e0cf3084fef", size = 51443229 }, + { url = "https://files.pythonhosted.org/packages/c8/15/89b0d616e4b6d92198a61019e7347a565a52b37ba3eed59178be00dcb8ab/timezonefinder-6.5.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4fa18e4b3f3ac1469bc50031700bc9696477cbf40953bdcbdec7bef20f3200d0", size = 51445199 }, + { url = "https://files.pythonhosted.org/packages/c9/aa/57fb0cb5b739a12ce6fc401b9d1e5d0b2ee092449c1d3ae14aa815409062/timezonefinder-6.5.9-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e9a0caf638f43b6dd9980731d1424500c7a6a5048db808aad7032560df5c663", size = 51443482 }, + { url = "https://files.pythonhosted.org/packages/2b/b9/c8a05e55096deea0b789d649da9729312ab1d97e3c52b0f0254c18f94cc7/timezonefinder-6.5.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c5347c4a73b40af4867a2946a5172ec68b644e7036888b9b5a0e568499bfc0f3", size = 51445521 }, + { url = "https://files.pythonhosted.org/packages/53/6c/498b3f453b15d7ecf2ecf54a8f1cc7fbd1d3c3eed86a5182081ebc17b180/timezonefinder-6.5.9-cp313-cp313-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d1f30b550c24598459643285ef0f5469524ad7a32dd854199e7a2a463600ff", size = 51443403 }, + { url = "https://files.pythonhosted.org/packages/8b/16/6ae92a93dd703c0485e4c0a76e28bc829f9c7eba548a1b2e17fb3342e4c3/timezonefinder-6.5.9-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:4f36877ad3a988f329cbd3f04f7cadc56dce073ad4a7bda7397f46ac2a61f9a4", size = 51445432 }, +] + [[package]] name = "tinycss2" version = "1.3.0" From 1765a92d797f4dc3b5102009e76be13ffbacf680 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sun, 1 Jun 2025 22:40:13 -0700 Subject: [PATCH 43/54] docs: update --- docs/source/application/wind_estimation.ipynb | 657 ------------------ .../tex/084c4d336ce7b83477dc39e2638fa8c5.svg | 13 - .../tex/142a0e55e7be4656275f11bc455886d4.svg | 11 - .../tex/191de8bb417c3c87a59bd9e2f4222ed0.svg | 11 - .../tex/42c4a538b0898977509d72e9843ffaad.svg | 43 -- .../tex/46f5f4c88affca88279076c9fd0b8502.svg | 30 - .../tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg | 21 - .../tex/547e731741446703f67244e558fc0508.svg | 13 - .../tex/7db7bb668a45f8c25263aa8d42c92f0e.svg | 11 - .../tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg | 42 -- .../tex/acbc3da08b00b7616f45697dcc62d44d.svg | 12 - .../tex/b215101f20292f1e8b0eb0ee28cf1d61.svg | 36 - .../tex/bd971e27e71d9bf4792553ddfe23d537.svg | 11 - .../tex/c39017bed62ee8b2a521779ae976ebe0.svg | 17 - .../wind_parameterizations.md | 44 -- src/geodata/datasets/__init__.py | 10 +- 16 files changed, 9 insertions(+), 973 deletions(-) delete mode 100644 docs/source/application/wind_estimation.ipynb delete mode 100644 docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg delete mode 100644 docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg delete mode 100644 docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg delete mode 100644 docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg delete mode 100644 docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg delete mode 100644 docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg delete mode 100644 docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg delete mode 100644 docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg delete mode 100644 docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg delete mode 100644 docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg delete mode 100644 docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg delete mode 100644 docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg delete mode 100644 docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg delete mode 100644 docs/source/parameterizations/wind_parameterizations.md diff --git a/docs/source/application/wind_estimation.ipynb b/docs/source/application/wind_estimation.ipynb deleted file mode 100644 index 82536b92..00000000 --- a/docs/source/application/wind_estimation.ipynb +++ /dev/null @@ -1,657 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Example: Use Geodata to Estimate Wind Speed and Direction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Geodata can be used for many applications. In this example, we will use the Geodata to estimate the\n", - "wind speed and direction at a given location." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import Necessary Packages" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "import geodata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def log_ratio(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic ratio law\n", - " Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " \"\"\"\n", - "\n", - " wnd_spd = ds[from_name] * (\n", - " np.log(to_height / ds[\"roughness\"]) / np.log(ds[from_height] / ds[\"roughness\"])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log ratio\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " \"\"\"\n", - " vonk = 0.4\n", - " wnd_spd = ds[from_name] + (ds[\"ustar\"] / vonk * np.log((to_height - ds[\"disph\"]) / ds[from_height]))\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "## Flux stability correction functions\n", - "def psi_linear(z, ds):\n", - " \"\"\"NOTE: This function does not perform well for high z/L\n", - " Linear stability correction [1,2]\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " Eval = \t0, if z/L <=0\n", - " linear if z/L > 0\n", - " [1] Businger, J.A., Wyngaard, J.C., Izumi, Y., Bradley, E.F., 1971. Flux profile relationships in the atmospheric surface layer. J. Atmos. Sci. 28, 181–189.\n", - " [2] Dyer, A.J., 1974. A review of flux–profile relationships. Boundary-Layer Meteorol. 7, 363–372.\n", - " \"\"\"\n", - " beta = 5.2\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[0 < ds[\"a\"]] = -beta * ds[\"a\"].values[0 < ds[\"a\"]]\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexp(z, ds):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2])\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexpconst(z, ds, const=7):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2]) with constant plateau\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - " const = upper bound of z/L, after which = constant\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] > const] = -A * (const + B * (const - C / D) * np.exp(-D * const) + B * C / D)\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def L_vph(ds):\n", - " \"\"\"Obuhkov length using virtual potential heat flux term [1] (described in detail in SI [2])\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " grav = 9.81 # gravitational acceleration in kg m s-2\n", - " CPD = 1004 # specific heat of dry air at constant pressure J K-1 kg-1\n", - " Le = 2.257e6 # latent heat of evaporation [J/kg]\n", - " kp = 2 / 7 # Poisson constant\n", - " Rd = 287 # Ideal gas constant [J/kg/K]\n", - " p0 = 1e5 # standard air pressure\n", - "\n", - " ds[\"p\"] = ds[\"rhoa\"] * Rd * ds[\"tlml\"]\n", - " ds[\"vphflux\"] = ds[\"hflux\"] + 0.61 * CPD / Le * ds[\"tlml\"] * (p0 / ds[\"p\"]) ** kp * ds[\"eflux\"]\n", - " ds[\"L\"] = -(ds[\"tlml\"] * ds[\"ustar\"] ** 3 * CPD * ds[\"rhoa\"]) / (vonk * grav * ds[\"vphflux\"])\n", - " return ds[\"L\"]\n", - "\n", - "\n", - "def winddir(ds):\n", - " \"\"\"Wind direction using lowest model layer\"\"\"\n", - "\n", - " ds[\"winddir\"] = np.degrees(np.arctan(ds[\"ulml\"] / ds[\"vlml\"]))\n", - " ds[\"winddir\"].values[ds[\"vlml\"] < 0] += 180\n", - " ds[\"winddir\"].values[(ds[\"vlml\"] > 0) & (ds[\"ulml\"] < 0)] += 360\n", - " return ds[\"winddir\"]\n", - "\n", - "\n", - "def _log_law_flux(ds, to_height, from_height, from_name, psifn, Lfn=L_vph): # pylint: disable=unused-argument\n", - " \"\"\"Compute logarithmic (integration) law given stability correction fn in terms of Obukhov length (derived from heat flux) [1]\n", - " Called by: log_law_flux_**\n", - "\n", - " [1] Sharan, M., & Aditi. (2009).\n", - " Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions.\n", - " Atmospheric Research, 94(2), 246–253.\n", - " https://doi.org/10.1016/j.atmosres.2009.05.014\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " ds[\"L\"] = L_vph(ds)\n", - "\n", - " wnd_spd = ds[from_name] + ds[\"ustar\"] / vonk * (\n", - " np.log((to_height - ds[\"disph\"]) / ds[from_height]) - psifn(to_height, ds[[\"L\", \"roughness\"]])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law and stability correction {psifn}\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law_flux_linear(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with linear stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linear)\n", - "\n", - "\n", - "def log_law_flux_linearexp(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexp)\n", - "\n", - "\n", - "def log_law_flux_linearexpconst(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential-constant stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexpconst)\n", - "\n", - "\n", - "\"\"\"\n", - "Main call (from convert.convert_wind)\n", - "\"\"\"\n", - "\n", - "\n", - "def extrapolate_wind_speed(ds, to_height, extrap_fn=log_ratio, from_height=None, var_height=None):\n", - " \"\"\"Extrapolate the wind speed from a given height above ground to another.\n", - "\n", - " If ds already contains a key refering to wind speeds at the desired to_height,\n", - " no conversion is done and the wind speeds are directly returned.\n", - "\n", - " Otherwise, extrapolates according to (1) extrap_fn and (2) heights\n", - "\n", - "\n", - "\n", - " Parameters\n", - " ----------\n", - " ds : xarray.Dataset\n", - " Dataset containing the wind speed time-series\n", - " to_height : int|float\n", - " Height (m) to which the wind speeds are extrapolated\n", - " extrap_fn : function for wind speed extrapolation\n", - " log_ratio : wind speed follows the logarithmic ratio law as desribed in [1]\n", - " log_law : wind speed follows logarithmic (integration) law described in [3]\n", - " power_law : wind speed follows power law (with fixed alpha), e.g., in [4]\n", - " from_height : int\n", - " (Optional)\n", - " Height (m) from which the wind speeds are interpolated to 'to_height'.\n", - " If not provided, the closest height to 'to_height' is selected.\n", - " var_height : str\n", - " (Optional)\n", - " suffix of variables in ds corresponding to variable height\n", - " e.g., `lml` => height contained in `hlml`, wind speed contained in `wndlml`\n", - "\n", - " Returns\n", - " -------\n", - " da : xarray.DataArray\n", - " DataArray containing the extrapolated wind speeds. Name of the DataArray\n", - " is 'wnd{to_height:d}'.\n", - "\n", - " References\n", - " ----------\n", - " [1] Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " [2] https://en.wikipedia.org/w/index.php?title=Roughness_length&oldid=862127433,\n", - " Retrieved 2019-02-15.\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " [4] Archer, C.L., Jacobson, M.Z., 2005. Evaluation of global wind power.\n", - " Journal of Geophysical Research 110, D12110.\n", - " \"\"\"\n", - "\n", - " to_name = \"wnd{h:0d}m\".format(h=int(to_height))\n", - " if to_name in ds:\n", - " # already found wind speed at given height in dataset\n", - " return ds[to_name]\n", - "\n", - " # Sanitize roughness for logarithm: 0.0002 corresponds to open water [2]\n", - " ds[\"roughness\"].values[ds[\"roughness\"].values <= 0.0] = 0.0002\n", - "\n", - " if not from_height is None:\n", - " # passed a from_height\n", - " if not var_height is None:\n", - " raise AssertionError(\"Cannot pass both from_height and var_height to extrapolate_wind_speed\")\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = wnd_spd.attrs[\"long_name\"] + \", \" + f\"from fixed height = {from_height}\"\n", - "\n", - " elif not var_height is None:\n", - " # passed a variable height (eg lml)\n", - " # set variable names\n", - " from_height = f\"h{var_height}\"\n", - " from_name = f\"wnd{var_height}\"\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, from_height, from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from variable height = {var_height}\"\n", - " )\n", - "\n", - " else:\n", - " # based on nearest height\n", - " heights = np.asarray([int(s[3:-1]) for s in ds if s.startswith(\"wnd\")])\n", - " if len(heights) == 0:\n", - " raise AssertionError(\"Wind speed is not in dataset\")\n", - "\n", - " from_height = heights[np.argmin(np.abs(heights - to_height))]\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from nearest height = {from_height}\"\n", - " )\n", - "\n", - " return wnd_spd.rename(to_name)\n", - "## Copyright 2020 Michael Davidson (UCSD).\n", - "\n", - "## This program is free software; you can redistribute it and/or\n", - "## modify it under the terms of the GNU General Public License as\n", - "## published by the Free Software Foundation; either version 3 of the\n", - "## License, or (at your option) any later version.\n", - "\n", - "## This program is distributed in the hope that it will be useful,\n", - "## but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "## GNU General Public License for more details.\n", - "\n", - "## You should have received a copy of the GNU General Public License\n", - "## along with this program. If not, see .\n", - "\n", - "\n", - "\"\"\"\n", - "GEODATA\n", - "\n", - "Geospatial Data Collection and \"Pre-Analysis\" Tools\n", - "\"\"\"\n", - "import logging\n", - "\n", - "import numpy as np\n", - "\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "\n", - "\"\"\"\n", - "Extrapolation functions\n", - "\tNote: these are called internally\n", - "\"\"\"\n", - "\n", - "\n", - "def log_ratio(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic ratio law\n", - " # Equation (2) in Andresen, G. et al (2015):\n", - " # \t'Validation of Danish wind time series from a new global renewable\n", - " # \tenergy atlas for energy system analysis'.\n", - " \"\"\"\n", - "\n", - " wnd_spd = ds[from_name] * (\n", - " np.log(to_height / ds[\"roughness\"]) / np.log(ds[from_height] / ds[\"roughness\"])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log ratio\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " \"\"\"\n", - " vonk = 0.4\n", - " wnd_spd = ds[from_name] + (ds[\"ustar\"] / vonk * np.log((to_height - ds[\"disph\"]) / ds[from_height]))\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "## Flux stability correction functions\n", - "def psi_linear(z, ds):\n", - " \"\"\"NOTE: This function does not perform well for high z/L\n", - " Linear stability correction [1,2]\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " Eval = \t0, if z/L <=0\n", - " linear if z/L > 0\n", - " [1] Businger, J.A., Wyngaard, J.C., Izumi, Y., Bradley, E.F., 1971. Flux profile relationships in the atmospheric surface layer. J. Atmos. Sci. 28, 181–189.\n", - " [2] Dyer, A.J., 1974. A review of flux–profile relationships. Boundary-Layer Meteorol. 7, 363–372.\n", - " \"\"\"\n", - " beta = 5.2\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[0 < ds[\"a\"]] = -beta * ds[\"a\"].values[0 < ds[\"a\"]]\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexp(z, ds):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2])\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexpconst(z, ds, const=7):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2]) with constant plateau\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - " const = upper bound of z/L, after which = constant\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] > const] = -A * (const + B * (const - C / D) * np.exp(-D * const) + B * C / D)\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def L_vph(ds):\n", - " \"\"\"Obuhkov length using virtual potential heat flux term [1] (described in detail in SI [2])\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " grav = 9.81 # gravitational acceleration in kg m s-2\n", - " CPD = 1004 # specific heat of dry air at constant pressure J K-1 kg-1\n", - " Le = 2.257e6 # latent heat of evaporation [J/kg]\n", - " kp = 2 / 7 # Poisson constant\n", - " Rd = 287 # Ideal gas constant [J/kg/K]\n", - " p0 = 1e5 # standard air pressure\n", - "\n", - " ds[\"p\"] = ds[\"rhoa\"] * Rd * ds[\"tlml\"]\n", - " ds[\"vphflux\"] = ds[\"hflux\"] + 0.61 * CPD / Le * ds[\"tlml\"] * (p0 / ds[\"p\"]) ** kp * ds[\"eflux\"]\n", - " ds[\"L\"] = -(ds[\"tlml\"] * ds[\"ustar\"] ** 3 * CPD * ds[\"rhoa\"]) / (vonk * grav * ds[\"vphflux\"])\n", - " return ds[\"L\"]\n", - "\n", - "\n", - "def winddir(ds):\n", - " \"\"\"Wind direction using lowest model layer\"\"\"\n", - "\n", - " ds[\"winddir\"] = np.degrees(np.arctan(ds[\"ulml\"] / ds[\"vlml\"]))\n", - " ds[\"winddir\"].values[ds[\"vlml\"] < 0] += 180\n", - " ds[\"winddir\"].values[(ds[\"vlml\"] > 0) & (ds[\"ulml\"] < 0)] += 360\n", - " return ds[\"winddir\"]\n", - "\n", - "\n", - "def _log_law_flux(ds, to_height, from_height, from_name, psifn, Lfn=L_vph): # pylint: disable=unused-argument\n", - " \"\"\"Compute logarithmic (integration) law given stability correction fn in terms of Obukhov length (derived from heat flux) [1]\n", - " Called by: log_law_flux_**\n", - "\n", - " [1] Sharan, M., & Aditi. (2009).\n", - " Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions.\n", - " Atmospheric Research, 94(2), 246–253.\n", - " https://doi.org/10.1016/j.atmosres.2009.05.014\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " ds[\"L\"] = L_vph(ds)\n", - "\n", - " wnd_spd = ds[from_name] + ds[\"ustar\"] / vonk * (\n", - " np.log((to_height - ds[\"disph\"]) / ds[from_height]) - psifn(to_height, ds[[\"L\", \"roughness\"]])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law and stability correction {psifn}\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law_flux_linear(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with linear stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linear)\n", - "\n", - "\n", - "def log_law_flux_linearexp(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexp)\n", - "\n", - "\n", - "def log_law_flux_linearexpconst(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential-constant stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexpconst)\n", - "\n", - "\n", - "\"\"\"\n", - "Main call (from convert.convert_wind)\n", - "\"\"\"\n", - "\n", - "\n", - "def extrapolate_wind_speed(ds, to_height, extrap_fn=log_ratio, from_height=None, var_height=None):\n", - " \"\"\"Extrapolate the wind speed from a given height above ground to another.\n", - "\n", - " If ds already contains a key refering to wind speeds at the desired to_height,\n", - " no conversion is done and the wind speeds are directly returned.\n", - "\n", - " Otherwise, extrapolates according to (1) extrap_fn and (2) heights\n", - "\n", - "\n", - "\n", - " Parameters\n", - " ----------\n", - " ds : xarray.Dataset\n", - " Dataset containing the wind speed time-series\n", - " to_height : int|float\n", - " Height (m) to which the wind speeds are extrapolated\n", - " extrap_fn : function for wind speed extrapolation\n", - " log_ratio : wind speed follows the logarithmic ratio law as desribed in [1]\n", - " log_law : wind speed follows logarithmic (integration) law described in [3]\n", - " power_law : wind speed follows power law (with fixed alpha), e.g., in [4]\n", - " from_height : int\n", - " (Optional)\n", - " Height (m) from which the wind speeds are interpolated to 'to_height'.\n", - " If not provided, the closest height to 'to_height' is selected.\n", - " var_height : str\n", - " (Optional)\n", - " suffix of variables in ds corresponding to variable height\n", - " e.g., `lml` => height contained in `hlml`, wind speed contained in `wndlml`\n", - "\n", - " Returns\n", - " -------\n", - " da : xarray.DataArray\n", - " DataArray containing the extrapolated wind speeds. Name of the DataArray\n", - " is 'wnd{to_height:d}'.\n", - "\n", - " References\n", - " ----------\n", - " [1] Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " [2] https://en.wikipedia.org/w/index.php?title=Roughness_length&oldid=862127433,\n", - " Retrieved 2019-02-15.\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " [4] Archer, C.L., Jacobson, M.Z., 2005. Evaluation of global wind power.\n", - " Journal of Geophysical Research 110, D12110.\n", - " \"\"\"\n", - "\n", - " to_name = \"wnd{h:0d}m\".format(h=int(to_height))\n", - " if to_name in ds:\n", - " # already found wind speed at given height in dataset\n", - " return ds[to_name]\n", - "\n", - " # Sanitize roughness for logarithm: 0.0002 corresponds to open water [2]\n", - " ds[\"roughness\"].values[ds[\"roughness\"].values <= 0.0] = 0.0002\n", - "\n", - " if not from_height is None:\n", - " # passed a from_height\n", - " if not var_height is None:\n", - " raise AssertionError(\"Cannot pass both from_height and var_height to extrapolate_wind_speed\")\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = wnd_spd.attrs[\"long_name\"] + \", \" + f\"from fixed height = {from_height}\"\n", - "\n", - " elif not var_height is None:\n", - " # passed a variable height (eg lml)\n", - " # set variable names\n", - " from_height = f\"h{var_height}\"\n", - " from_name = f\"wnd{var_height}\"\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, from_height, from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from variable height = {var_height}\"\n", - " )\n", - "\n", - " else:\n", - " # based on nearest height\n", - " heights = np.asarray([int(s[3:-1]) for s in ds if s.startswith(\"wnd\")])\n", - " if len(heights) == 0:\n", - " raise AssertionError(\"Wind speed is not in dataset\")\n", - "\n", - " from_height = heights[np.argmin(np.abs(heights - to_height))]\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from nearest height = {from_height}\"\n", - " )\n", - "\n", - " return wnd_spd.rename(to_name)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "geodata", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg b/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg deleted file mode 100644 index 1b3dc80d..00000000 --- a/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg b/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg deleted file mode 100644 index 36f95779..00000000 --- a/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg b/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg deleted file mode 100644 index 5ea91e8f..00000000 --- a/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg b/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg deleted file mode 100644 index b152ece5..00000000 --- a/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg b/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg deleted file mode 100644 index 38e293a5..00000000 --- a/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg b/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg deleted file mode 100644 index 11862566..00000000 --- a/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg b/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg deleted file mode 100644 index b2afdbcc..00000000 --- a/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg b/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg deleted file mode 100644 index e5f2c7be..00000000 --- a/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg b/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg deleted file mode 100644 index 7474165f..00000000 --- a/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg b/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg deleted file mode 100644 index bb9bebf7..00000000 --- a/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg b/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg deleted file mode 100644 index d1b43f3f..00000000 --- a/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg b/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg deleted file mode 100644 index 657839b2..00000000 --- a/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg b/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg deleted file mode 100644 index ebad50a2..00000000 --- a/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/wind_parameterizations.md b/docs/source/parameterizations/wind_parameterizations.md deleted file mode 100644 index 2f9399f7..00000000 --- a/docs/source/parameterizations/wind_parameterizations.md +++ /dev/null @@ -1,44 +0,0 @@ -# Wind Extrapolation and Stability Correction - -## Extrapolation - -Extrapolating wind speeds from measured (or modeled) heights to other heights must assume a model of the atmosphere. This is particularly important for studying wind power generation as turbine hub heights can be 50-120m above the ground. Standard approaches include the log law and power law. - -Geodata allows users to customize the wind extrapolation function through the `extrap_fn` parameter in the call to `wind.extrapolate_wind_speed()`. By default, the log law ratio (`extrap_fn = log_ratio`) is used. - - -## Stability correction - -Simple extrapolation routines assume neutral stability of the atmosphere, potentially ignoring highly stable or unstable atmospheric conditions that could alter the wind speed profile at certain hours of the day. - - -**`L` Obukhov length** - -Most stability correction functions rely on the Obukhov length, which can be calculated from available parameters [[1]](#references): - -$L= - \dfrac{u_{* }^{3}\overline{\theta_{v}}\rho_{a}C_p}{\kappa gH_{v0}}$ -$H_{v0}=H_{f}+0.61C_{p}\dfrac{\overline{\theta}}{L_{e}}H_{L}$ -$\overline{\theta}=\overline{T}\left(p_{0}/p\right)^{\kappa_{p}}$ - -where $u_{* }$ is the friction velocity, $\overline{\theta_{v}}$ is the virtual temperature, $\rho_a$ is the density, $C_p$ is the specific heat, $\kappa=0.4$, and $H_{v0}$ is the virtual heat flux in terms of sensible and latent heat fluxes (negative if directed upwards). - - -**$\psi_m$ Stability correction** - -Extrapolated wind speeds are "corrected" via a stability correction function $\psi_m$ according to: - -$u'(z) = u(z) - \psi_m(z/L)$ - -where $\psi_m(z/L)$ takes different forms in the literature [[2]](#references). - -Geodata has the following stability correction functions installed (in terms of the parameter $z/L$): -- `psi_linear`: linear for positive parameters (This function does not perform well for high $z/L$) -- `psi_linearexp`: piecewise linear-exponential -- `psi_linearexpconst`: piecewise linear-exponential with maximum constant correction, illustrated here for various constants: -![Stability Correction](../_static/images/wind_stability_corrections.png) - - -## References - -- [1] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165. https://doi.org/10.1016/j.renene.2016.03.028 -- [2] Sharan, M., & Aditi. (2009). Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions. Atmospheric Research, 94(2), 246–253. https://doi.org/10.1016/j.atmosres.2009.05.014 diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index 80023216..f6c7defc 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -15,6 +15,13 @@ # along with this program. If not, see . +"""A interface to download and manage geospatial datasets. + +Attributes: + registry (dict): Registry of available datasets. You can retrieve relevant dataset classes using `registry.get()`. + DatasetType (type): A type to aid type hinting dataset classes. +""" + from . import era5, merra2 from ._base import DatasetType from ._base import _registry as registry @@ -23,5 +30,6 @@ def register_hrrr(): - """Register the HRRR dataset with the registry.""" + """Register the HRRR dataset with the registry. By default, the HRRR dataset is not registered to avoid import overhead with the HRRR Herbie package. + This function can be called to register the HRRR dataset when needed.""" from . import hrrr # noqa: F401 From 29b9455b6257bcaa9f6500e19f3e801f4133f802 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:02:53 -0700 Subject: [PATCH 44/54] docs: add draft docs for dataset module --- docs/source/datasets/overview.rst | 89 +++++++++++++++++++++++++++++++ pyproject.toml | 3 +- uv.lock | 51 ++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 docs/source/datasets/overview.rst diff --git a/docs/source/datasets/overview.rst b/docs/source/datasets/overview.rst new file mode 100644 index 00000000..d4ff5f64 --- /dev/null +++ b/docs/source/datasets/overview.rst @@ -0,0 +1,89 @@ +========================== +Dataset Module Overview +========================== + +The ``geodata.datasets`` module provides tools and classes for accessing, managing, and +processing geospatial datasets. It offers a unified interface for loading various +data formats, handling metadata, and performing common geospatial operations. + +Key Features +------------ + +- Supports the download and management of datasets from various sources, such as + `ERA5 `_ and + `MERRA2 `. + +- Provides a consistent API for accessing geospatial data, regardless of the underlying + data source. + +Typical Usage +------------- + +In the following example, we will demonstrate how to download a dataset containing wind +and solar data from ECMWF's ERA5 dataset. + +.. code-block:: python + + from geodata.datasets import load_dataset + + dataset_cls = load_dataset("wind_solar_hourly") + + years = slice(2010, 2020) + months = slice(1, 13) + dataset = dataset_cls(years=years, months=months) + +Here, we first create a dataset class using the `load_dataset` function, specifying the +name of the dataset we want to load. We then instantiate the dataset class with the +desired time range (years and months). Then, we can create a dataset instance with +that class, which will handle the downloading and processing of the data. + +Dataset Classes +----------------- +The `geodata.datasets` module includes several dataset classes, each tailored for +specific datasets. These classes encapsulate the logic for downloading, processing, and +accessing the data. Some of the available dataset classes +(listed by `weather_data_config`) include: + +- `wind_solar_hourly`: A dataset containing hourly wind and solar data from ECMWF's + ERA5. It is important to note that the wind data are only recorded at + 10 and 100 meters above ground level. Hence, this dataset is also referred to as + 2D wind and solar dataset. + +- `wind_3d_hourly`: A dataset containing hourly wind data from ECMWF's ERA5 at + multiple vertical levels, providing a more comprehensive view of the wind profile. + It can be used for wind speed estimation using and interpolation model built into + the geodata library. + +You can use the `list_datasets` function to see all available datasets in the +`geodata.datasets` module. This function returns a list of dataset names that can be +loaded using the `load_dataset` function. For example: + +.. code-block:: python + + from geodata.datasets import list_datasets + + available_datasets = list_datasets() + print(available_datasets) # Outputs a list of available dataset names. + +Check Preparedness of Datasets +------------------------------------------------ +To check if a dataset is prepared and ready for use, you can use the `downloaded` +property of the dataset instance. This property returns a boolean indicating whether the +dataset is fully prepared. If the dataset is not prepared, you can call the `prepare` +method to download and process the data. For example: + +.. code-block:: python + + print(dataset.downloaded) # Check if the dataset is downloaded. Outputs False here. + + if not dataset.downloaded: + dataset.download() + + print(dataset.downloaded) # Outputs True after downloading. + +Dataset's Interoperability with Cutout +------------------------------------------------ + +At the moment, the dataset classes are not interoperable with the `Cutout` class. +In the future, we plan to consolidate the functionalities of the `Cutout` class into the +dataset classes and the modeling module (see :doc:`here<../modeling/wind/index>`). diff --git a/pyproject.toml b/pyproject.toml index a5243667..01122e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,8 @@ docs = [ "sphinx>=8.0.0", "myst-nb>=1.1.2", "sphinx-book-theme>=1.1.3", - "sphinx-autoapi==3.3.2" + "sphinx-autoapi==3.3.2", + "doc8>=1.1.2", ] accelerate = [ "numba>=0.61.0", diff --git a/uv.lock b/uv.lock index 9cd0bc17..b248a433 100644 --- a/uv.lock +++ b/uv.lock @@ -664,6 +664,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] +[[package]] +name = "doc8" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "pygments" }, + { name = "restructuredtext-lint" }, + { name = "stevedore" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/28/b0a576233730b756ca1ebb422bc6199a761b826b86e93e5196dfa85331ea/doc8-1.1.2.tar.gz", hash = "sha256:1225f30144e1cc97e388dbaf7fe3e996d2897473a53a6dae268ddde21c354b98", size = 27030 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/f1/6ffd5d76578e98a8f21ae7216b88a7212c778f665f1a8f4f8ce6f9605da4/doc8-1.1.2-py3-none-any.whl", hash = "sha256:e787b3076b391b8b49400da5d018bacafe592dfc0a04f35a9be22d0122b82b59", size = 25794 }, +] + [[package]] name = "docutils" version = "0.21.2" @@ -824,6 +840,7 @@ accelerate = [ { name = "numba" }, ] docs = [ + { name = "doc8" }, { name = "myst-nb" }, { name = "sphinx" }, { name = "sphinx-autoapi" }, @@ -851,6 +868,7 @@ requires-dist = [ { name = "bottleneck", specifier = ">=1.3.6" }, { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.5" }, { name = "dask", specifier = ">=2024.9.0" }, + { name = "doc8", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "geopandas", specifier = ">=1.0.1" }, { name = "h5netcdf", specifier = ">=1.6.1" }, { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2024.8.0" }, @@ -2095,6 +2113,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, ] +[[package]] +name = "pbr" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/d2/510cc0d218e753ba62a1bc1434651db3cd797a9716a0a66cc714cb4f0935/pbr-6.1.1.tar.gz", hash = "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b", size = 125702 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/ac/684d71315abc7b1214d59304e23a982472967f6bf4bde5a98f1503f648dc/pbr-6.1.1-py2.py3-none-any.whl", hash = "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76", size = 108997 }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -2644,6 +2674,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, ] +[[package]] +name = "restructuredtext-lint" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/9c/6d8035cafa2d2d314f34e6cd9313a299de095b26e96f1c7312878f988eec/restructuredtext_lint-1.4.0.tar.gz", hash = "sha256:1b235c0c922341ab6c530390892eb9e92f90b9b75046063e047cacfb0f050c45", size = 16723 } + [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -3091,6 +3130,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521 }, ] +[[package]] +name = "stevedore" +version = "5.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pbr" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/3f/13cacea96900bbd31bb05c6b74135f85d15564fc583802be56976c940470/stevedore-5.4.1.tar.gz", hash = "sha256:3135b5ae50fe12816ef291baff420acb727fcd356106e3e9cbfa9e5985cd6f4b", size = 513858 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/45/8c4ebc0c460e6ec38e62ab245ad3c7fc10b210116cea7c16d61602aa9558/stevedore-5.4.1-py3-none-any.whl", hash = "sha256:d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe", size = 49533 }, +] + [[package]] name = "tabulate" version = "0.9.0" From 2cdfa3d7a7e340d1b2c18a5846e3ef118f0aee2e Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 6 Jun 2025 11:12:55 -0700 Subject: [PATCH 45/54] fix: update herbie version to avoid crashing on cluster --- pyproject.toml | 6 +- uv.lock | 220 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 163 insertions(+), 63 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 01122e80..bab31149 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "netcdf4>=1.7.1.post2", "boto3==1.26.46", "toolz>=0.12.1", - "pyproj==3.6.1", "requests>=2.32.3", "matplotlib==3.9.2", "rasterio==1.4.0", @@ -28,11 +27,12 @@ dependencies = [ "shapely>=2.0.6", "geopandas>=1.0.1", "pyyaml>=6.0.2", - "dask>=2024.9.0", + "dask[distributed]>=2024.9.0", "tqdm>=4.66.5", "h5netcdf>=1.6.1", "pvlib>=0.12.0", "timezonefinder>=6.5.9", + "pyproj>=3.6.1", ] requires-python = ">=3.10" readme = "README.md" @@ -42,7 +42,7 @@ license = {text = "GPLv3"} [project.optional-dependencies] download = [ "cdsapi>=0.7.5", - "herbie-data>=2024.8.0", + "herbie-data>=2025.5.0", ] notebook = [ "notebook>=7.2.2", diff --git a/uv.lock b/uv.lock index b248a433..64bcef3f 100644 --- a/uv.lock +++ b/uv.lock @@ -358,7 +358,7 @@ wheels = [ [[package]] name = "cfgrib" -version = "0.9.14.1" +version = "0.9.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -366,9 +366,9 @@ dependencies = [ { name = "eccodes" }, { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/d9/274599a790dfc384d0a06a849adfbed0c924ec5376eda189e503325e7e3f/cfgrib-0.9.14.1.tar.gz", hash = "sha256:a6e66e8a3d8f9823d3eef0c2c6ebca602d5bcc324f0baf4f3d13f68b0b40501e", size = 6510867 } +sdist = { url = "https://files.pythonhosted.org/packages/3e/3b/0ccbbc67866a4a2df570d6bf0f53d6d22220c44e1f3684455b5eae298936/cfgrib-0.9.15.0.tar.gz", hash = "sha256:d455034e19b9560a75d008ba9d09b2d4e65762adfb2e911f28b841f4b9c6b47f", size = 6511752 } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/c6/7e8a2b07d2404a79b2cea577962f88e02662ba2b85e073b6e5eed8081878/cfgrib-0.9.14.1-py3-none-any.whl", hash = "sha256:0714ece262231b0d4006fc7ba5a04f287a9fd42473ac3f6ed4703eb2e7e92161", size = 48681 }, + { url = "https://files.pythonhosted.org/packages/7d/d7/96b4209c99f1fd6c19f502cebe8c91983c23331c380f3f521250f268ae8c/cfgrib-0.9.15.0-py3-none-any.whl", hash = "sha256:469cfd25dc173863795e596263b3b6b5ea1402b1715f2b7b1d4b995b40b32c18", size = 48908 }, ] [[package]] @@ -610,6 +610,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ea/3df533d551bc673f8a295450a8e28707e980fd3b55117edb1d1aa4cc374d/dask-2024.9.1-py3-none-any.whl", hash = "sha256:3757bb6c976f0436fef6bd6ad32f8983ee5ce7d8a738a1f643e208cd390ec794", size = 1257378 }, ] +[package.optional-dependencies] +distributed = [ + { name = "distributed" }, +] + [[package]] name = "datapi" version = "0.1.1" @@ -664,6 +669,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] +[[package]] +name = "distributed" +version = "2024.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "cloudpickle" }, + { name = "dask" }, + { name = "jinja2" }, + { name = "locket" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "sortedcontainers" }, + { name = "tblib" }, + { name = "toolz" }, + { name = "tornado" }, + { name = "urllib3" }, + { name = "zict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/da/bf22bb58ddfe518a4bc2385b5ec17a0ad1b7cd6bc2fa8f4310181da802a2/distributed-2024.9.1.tar.gz", hash = "sha256:4d573d89ff4fdde0dd96ad5cfdb843ce8ecef8caf002435bc60d14414dc1e819", size = 1113189 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ff/7531b2bebc84766905d6b6c636f21e465511f330957bc1632d00c75ac152/distributed-2024.9.1-py3-none-any.whl", hash = "sha256:f7acca78afcc50eb337fd46247a33efa5a1696b937083a12556ba33f11be1c74", size = 1020546 }, +] + [[package]] name = "doc8" version = "1.1.2" @@ -813,7 +844,7 @@ source = { editable = "." } dependencies = [ { name = "boto3" }, { name = "bottleneck" }, - { name = "dask" }, + { name = "dask", extra = ["distributed"] }, { name = "geopandas" }, { name = "h5netcdf" }, { name = "matplotlib" }, @@ -867,11 +898,11 @@ requires-dist = [ { name = "boto3", specifier = "==1.26.46" }, { name = "bottleneck", specifier = ">=1.3.6" }, { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.5" }, - { name = "dask", specifier = ">=2024.9.0" }, + { name = "dask", extras = ["distributed"], specifier = ">=2024.9.0" }, { name = "doc8", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "geopandas", specifier = ">=1.0.1" }, { name = "h5netcdf", specifier = ">=1.6.1" }, - { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2024.8.0" }, + { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2025.5.0" }, { name = "matplotlib", specifier = "==3.9.2" }, { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "netcdf4", specifier = ">=1.7.1.post2" }, @@ -881,7 +912,7 @@ requires-dist = [ { name = "numpy", specifier = "<2" }, { name = "pandas", specifier = ">=2.2.3" }, { name = "pvlib", specifier = ">=0.12.0" }, - { name = "pyproj", specifier = "==3.6.1" }, + { name = "pyproj", specifier = ">=3.6.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rasterio", specifier = "==1.4.0" }, { name = "requests", specifier = ">=2.32.3" }, @@ -1061,20 +1092,21 @@ wheels = [ [[package]] name = "herbie-data" -version = "2024.8.0" +version = "2025.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cfgrib" }, + { name = "eccodes" }, { name = "numpy" }, { name = "pandas" }, - { name = "pygrib" }, + { name = "pyproj" }, { name = "requests" }, { name = "toml" }, { name = "xarray" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/48/59/136b443a12073546ad7987db6d78dd43d8550689134145be4bbfac0d0a57/herbie_data-2024.8.0.tar.gz", hash = "sha256:83831205ea415b6f245d829cc13a162263074035e5e365c38f239ee04e3d6c14", size = 102912 } +sdist = { url = "https://files.pythonhosted.org/packages/df/91/cb0e1effccf51daffb615393d114a8110a466c15d41820a1d242294a2481/herbie_data-2025.5.0.tar.gz", hash = "sha256:0629374ae29d0ece5f58c9369219ed95ec262ac0544d6ecb7824e79fdc3a2aef", size = 9418650 } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0a/9bd4412d9fe1c30e26b58f4dbec64d39276110f84bf6316d12242b2f1296/herbie_data-2024.8.0-py3-none-any.whl", hash = "sha256:196ecc028dca71c99ffb7452d8a443a64b57c27605f70ac08f5810f3d606088b", size = 100959 }, + { url = "https://files.pythonhosted.org/packages/cf/47/8f8a5af62c941c5b7e415a8735d5c81f51943b0463b91974e269317e9038/herbie_data-2025.5.0-py3-none-any.whl", hash = "sha256:57eb86b456d626cd057aca42a4b1c3c4bf11a3c10ae0696e47e52891c82b4263", size = 113296 }, ] [[package]] @@ -1731,6 +1763,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/74/c95adcdf032956d9ef6c89a9b8a5152bf73915f8c633f3e3d88d06bd699c/mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205", size = 47958 }, ] +[[package]] +name = "msgpack" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d0/7555686ae7ff5731205df1012ede15dd9d927f6227ea151e901c7406af4f/msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e", size = 167260 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/f9/a892a6038c861fa849b11a2bb0502c07bc698ab6ea53359e5771397d883b/msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd", size = 150428 }, + { url = "https://files.pythonhosted.org/packages/df/7a/d174cc6a3b6bb85556e6a046d3193294a92f9a8e583cdbd46dc8a1d7e7f4/msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d", size = 84131 }, + { url = "https://files.pythonhosted.org/packages/08/52/bf4fbf72f897a23a56b822997a72c16de07d8d56d7bf273242f884055682/msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5", size = 81215 }, + { url = "https://files.pythonhosted.org/packages/02/95/dc0044b439b518236aaf012da4677c1b8183ce388411ad1b1e63c32d8979/msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5", size = 371229 }, + { url = "https://files.pythonhosted.org/packages/ff/75/09081792db60470bef19d9c2be89f024d366b1e1973c197bb59e6aabc647/msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e", size = 378034 }, + { url = "https://files.pythonhosted.org/packages/32/d3/c152e0c55fead87dd948d4b29879b0f14feeeec92ef1fd2ec21b107c3f49/msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b", size = 363070 }, + { url = "https://files.pythonhosted.org/packages/d9/2c/82e73506dd55f9e43ac8aa007c9dd088c6f0de2aa19e8f7330e6a65879fc/msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f", size = 359863 }, + { url = "https://files.pythonhosted.org/packages/cb/a0/3d093b248837094220e1edc9ec4337de3443b1cfeeb6e0896af8ccc4cc7a/msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68", size = 368166 }, + { url = "https://files.pythonhosted.org/packages/e4/13/7646f14f06838b406cf5a6ddbb7e8dc78b4996d891ab3b93c33d1ccc8678/msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b", size = 370105 }, + { url = "https://files.pythonhosted.org/packages/67/fa/dbbd2443e4578e165192dabbc6a22c0812cda2649261b1264ff515f19f15/msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044", size = 68513 }, + { url = "https://files.pythonhosted.org/packages/24/ce/c2c8fbf0ded750cb63cbcbb61bc1f2dfd69e16dca30a8af8ba80ec182dcd/msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f", size = 74687 }, + { url = "https://files.pythonhosted.org/packages/b7/5e/a4c7154ba65d93be91f2f1e55f90e76c5f91ccadc7efc4341e6f04c8647f/msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7", size = 150803 }, + { url = "https://files.pythonhosted.org/packages/60/c2/687684164698f1d51c41778c838d854965dd284a4b9d3a44beba9265c931/msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa", size = 84343 }, + { url = "https://files.pythonhosted.org/packages/42/ae/d3adea9bb4a1342763556078b5765e666f8fdf242e00f3f6657380920972/msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701", size = 81408 }, + { url = "https://files.pythonhosted.org/packages/dc/17/6313325a6ff40ce9c3207293aee3ba50104aed6c2c1559d20d09e5c1ff54/msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6", size = 396096 }, + { url = "https://files.pythonhosted.org/packages/a8/a1/ad7b84b91ab5a324e707f4c9761633e357820b011a01e34ce658c1dda7cc/msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59", size = 403671 }, + { url = "https://files.pythonhosted.org/packages/bb/0b/fd5b7c0b308bbf1831df0ca04ec76fe2f5bf6319833646b0a4bd5e9dc76d/msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0", size = 387414 }, + { url = "https://files.pythonhosted.org/packages/f0/03/ff8233b7c6e9929a1f5da3c7860eccd847e2523ca2de0d8ef4878d354cfa/msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e", size = 383759 }, + { url = "https://files.pythonhosted.org/packages/1f/1b/eb82e1fed5a16dddd9bc75f0854b6e2fe86c0259c4353666d7fab37d39f4/msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6", size = 394405 }, + { url = "https://files.pythonhosted.org/packages/90/2e/962c6004e373d54ecf33d695fb1402f99b51832631e37c49273cc564ffc5/msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5", size = 396041 }, + { url = "https://files.pythonhosted.org/packages/f8/20/6e03342f629474414860c48aeffcc2f7f50ddaf351d95f20c3f1c67399a8/msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88", size = 68538 }, + { url = "https://files.pythonhosted.org/packages/aa/c4/5a582fc9a87991a3e6f6800e9bb2f3c82972912235eb9539954f3e9997c7/msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788", size = 74871 }, + { url = "https://files.pythonhosted.org/packages/e1/d6/716b7ca1dbde63290d2973d22bbef1b5032ca634c3ff4384a958ec3f093a/msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d", size = 152421 }, + { url = "https://files.pythonhosted.org/packages/70/da/5312b067f6773429cec2f8f08b021c06af416bba340c912c2ec778539ed6/msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2", size = 85277 }, + { url = "https://files.pythonhosted.org/packages/28/51/da7f3ae4462e8bb98af0d5bdf2707f1b8c65a0d4f496e46b6afb06cbc286/msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420", size = 82222 }, + { url = "https://files.pythonhosted.org/packages/33/af/dc95c4b2a49cff17ce47611ca9ba218198806cad7796c0b01d1e332c86bb/msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2", size = 392971 }, + { url = "https://files.pythonhosted.org/packages/f1/54/65af8de681fa8255402c80eda2a501ba467921d5a7a028c9c22a2c2eedb5/msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39", size = 401403 }, + { url = "https://files.pythonhosted.org/packages/97/8c/e333690777bd33919ab7024269dc3c41c76ef5137b211d776fbb404bfead/msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f", size = 385356 }, + { url = "https://files.pythonhosted.org/packages/57/52/406795ba478dc1c890559dd4e89280fa86506608a28ccf3a72fbf45df9f5/msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247", size = 383028 }, + { url = "https://files.pythonhosted.org/packages/e7/69/053b6549bf90a3acadcd8232eae03e2fefc87f066a5b9fbb37e2e608859f/msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c", size = 391100 }, + { url = "https://files.pythonhosted.org/packages/23/f0/d4101d4da054f04274995ddc4086c2715d9b93111eb9ed49686c0f7ccc8a/msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b", size = 394254 }, + { url = "https://files.pythonhosted.org/packages/1c/12/cf07458f35d0d775ff3a2dc5559fa2e1fcd06c46f1ef510e594ebefdca01/msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b", size = 69085 }, + { url = "https://files.pythonhosted.org/packages/73/80/2708a4641f7d553a63bc934a3eb7214806b5b39d200133ca7f7afb0a53e8/msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f", size = 75347 }, + { url = "https://files.pythonhosted.org/packages/c8/b0/380f5f639543a4ac413e969109978feb1f3c66e931068f91ab6ab0f8be00/msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf", size = 151142 }, + { url = "https://files.pythonhosted.org/packages/c8/ee/be57e9702400a6cb2606883d55b05784fada898dfc7fd12608ab1fdb054e/msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330", size = 84523 }, + { url = "https://files.pythonhosted.org/packages/7e/3a/2919f63acca3c119565449681ad08a2f84b2171ddfcff1dba6959db2cceb/msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734", size = 81556 }, + { url = "https://files.pythonhosted.org/packages/7c/43/a11113d9e5c1498c145a8925768ea2d5fce7cbab15c99cda655aa09947ed/msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e", size = 392105 }, + { url = "https://files.pythonhosted.org/packages/2d/7b/2c1d74ca6c94f70a1add74a8393a0138172207dc5de6fc6269483519d048/msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca", size = 399979 }, + { url = "https://files.pythonhosted.org/packages/82/8c/cf64ae518c7b8efc763ca1f1348a96f0e37150061e777a8ea5430b413a74/msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915", size = 383816 }, + { url = "https://files.pythonhosted.org/packages/69/86/a847ef7a0f5ef3fa94ae20f52a4cacf596a4e4a010197fbcc27744eb9a83/msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d", size = 380973 }, + { url = "https://files.pythonhosted.org/packages/aa/90/c74cf6e1126faa93185d3b830ee97246ecc4fe12cf9d2d31318ee4246994/msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434", size = 387435 }, + { url = "https://files.pythonhosted.org/packages/7a/40/631c238f1f338eb09f4acb0f34ab5862c4e9d7eda11c1b685471a4c5ea37/msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c", size = 399082 }, + { url = "https://files.pythonhosted.org/packages/e9/1b/fa8a952be252a1555ed39f97c06778e3aeb9123aa4cccc0fd2acd0b4e315/msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc", size = 69037 }, + { url = "https://files.pythonhosted.org/packages/b6/bc/8bd826dd03e022153bfa1766dcdec4976d6c818865ed54223d71f07862b3/msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f", size = 75140 }, +] + [[package]] name = "multiurl" version = "0.3.3" @@ -2322,31 +2406,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513 }, ] -[[package]] -name = "pygrib" -version = "2.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "packaging" }, - { name = "pyproj" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/60/bac29fc06197f85efccb346879da3ae9ee125525d43e28b0a1b768831a74/pygrib-2.1.6.tar.gz", hash = "sha256:047980aeb010ef457999950bcc8e46556910316cb77fe78c0bd1b3520aa920f0", size = 21808824 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/11/55388ae3f0abce352942409ee687d74348fd86f54b60a9517f2311194fc3/pygrib-2.1.6-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:354028fcfc5f29dbebe16da2df3c8cf41383818cf27a43adfa6cc1a4e43c249b", size = 18472537 }, - { url = "https://files.pythonhosted.org/packages/ec/33/9395540f48099d6b04e9f583cc5428ebcb56d54beffbed9207d671d14826/pygrib-2.1.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:06b3fb25889225a599a908fdb8d4590ab30a2738048d877725f90b6a2a777983", size = 18358507 }, - { url = "https://files.pythonhosted.org/packages/70/4e/741e1f5fa63a08a98de12bf00f274249cd5c3f9296af11f11c16e4375d06/pygrib-2.1.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37ce2c761489b68f2642c739a488470b58aacf13c96bdae671dd35730960e94e", size = 18638885 }, - { url = "https://files.pythonhosted.org/packages/50/5d/12ae450349ce3d51844e775dff0ee1b04a9aee963d6ed6102ced290805b4/pygrib-2.1.6-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:d6b912cc528c3a87e4f9f990d5c1e57a54f61835dad5b8daac6adb048faf5c5a", size = 18473599 }, - { url = "https://files.pythonhosted.org/packages/53/9a/a1742fa64b2702d7723c2595bf1b36b1c56fd1915f6e3f185f057a79fbc3/pygrib-2.1.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:dc7fcb20bd5e2a94e1b7501bed45d418eadc6e325bed5f4de37a254409b5a5fe", size = 18358366 }, - { url = "https://files.pythonhosted.org/packages/c8/f0/9490a3bf86feef1be2678a6ce15131d44f6f06c863c2462f7f4a52c35c0a/pygrib-2.1.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69a42b44ae395280b08f62e598b696444446759ae4954db42ad2d01ee2c94d8b", size = 18633622 }, - { url = "https://files.pythonhosted.org/packages/71/99/436f5af4e9093277b13153c6ce51336df886e7493076c3d756c44c988d47/pygrib-2.1.6-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:fa4613eef98b26bf9a8719869f656aa15362f700311aec777b2190deb1060a1d", size = 18467761 }, - { url = "https://files.pythonhosted.org/packages/b0/4d/e2d3150961801d46af88c4cf113c00ff0825206ac6d726cec11f667c9f8e/pygrib-2.1.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:332d29bbc16749375b24c8d9bd01180c0a9eda471df86b2073432fba3339d658", size = 18356620 }, - { url = "https://files.pythonhosted.org/packages/ab/cc/4b2b85241086a3c42fcf4e9e225a1b2f5bfcb299a4c9c8e46f56c6a2d7c3/pygrib-2.1.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d368c359acacae4061ebcdfe635fb826f21fbc1f6c2227a269c870d9e8b49e70", size = 18617948 }, - { url = "https://files.pythonhosted.org/packages/ac/bb/a35a9b012c234416100abe53661cb7002e8fd6f3120f4f85475604cf9d00/pygrib-2.1.6-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:3cf2d87ad30449726d9279ca8cfa4afbdc83670750a4dfb5d965e3fc55d3aa8d", size = 18466301 }, - { url = "https://files.pythonhosted.org/packages/fd/15/bb162b016378993b5a779d83ec499dfd82d8df7776885fb826b69d2b7b98/pygrib-2.1.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:00f84e99372c6cf51f6a67e26c394dc3371ec356906e5aa9177f12372d92b228", size = 18356130 }, - { url = "https://files.pythonhosted.org/packages/ea/42/112c2f6836e730343fe21ad85e916c1b742d1f951738a3e9b3f6fab65128/pygrib-2.1.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56d70f492e1a298429a94662c5003c6959d8cc1b078173312fa9cf3bf36a14b8", size = 18613954 }, -] - [[package]] name = "pyogrio" version = "0.10.0" @@ -2395,31 +2454,45 @@ wheels = [ [[package]] name = "pyproj" -version = "3.6.1" +version = "3.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/84/2b39bbf888c753ea48b40d47511548c77aa03445465c35cc4c4e9649b643/pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf", size = 225131 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/32/63cf474f4a8d4804b3bdf7c16b8589f38142e8e2f8319dcea27e0bc21a87/pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4", size = 6142763 }, - { url = "https://files.pythonhosted.org/packages/18/86/2e7cb9de40492f1bafbf11f4c9072edc394509a40b5e4c52f8139546f039/pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe", size = 4877123 }, - { url = "https://files.pythonhosted.org/packages/5e/c5/928d5a26995dbefbebd7507d982141cd9153bc7e4392b334fff722c4af12/pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f", size = 6190576 }, - { url = "https://files.pythonhosted.org/packages/f6/2b/b60cf73b0720abca313bfffef34e34f7f7dae23852b2853cf0368d49426b/pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35", size = 8328075 }, - { url = "https://files.pythonhosted.org/packages/d9/a8/7193f46032636be917bc775506ae987aad72c931b1f691b775ca812a2917/pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f", size = 5635713 }, - { url = "https://files.pythonhosted.org/packages/89/8f/27350c8fba71a37cd0d316f100fbd96bf139cc2b5ff1ab0dcbc7ac64010a/pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e", size = 6087932 }, - { url = "https://files.pythonhosted.org/packages/84/a6/a300c1b14b2112e966e9f90b18f9c13b586bdcf417207cee913ae9005da3/pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882", size = 6147442 }, - { url = "https://files.pythonhosted.org/packages/30/bd/b9bd3761f08754e8dbb34c5a647db2099b348ab5da338e90980caf280e37/pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a", size = 4880331 }, - { url = "https://files.pythonhosted.org/packages/f4/0a/d82aeeb605b5d6870bc72307c3b5e044e632eb7720df8885e144f51a8eac/pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85", size = 6192425 }, - { url = "https://files.pythonhosted.org/packages/64/90/dfe5c00de1ca4dbb82606e79790659d4ed7f0ed8d372bccb3baca2a5abe0/pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915", size = 8571478 }, - { url = "https://files.pythonhosted.org/packages/14/6d/ae373629a1723f0db80d7b8c93598b00d9ecb930ed9ebf4f35826a33e97c/pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132", size = 5634575 }, - { url = "https://files.pythonhosted.org/packages/79/95/eb68113c5b5737c342bde1bab92705dabe69c16299c5a122616e50f1fbd6/pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2", size = 6088494 }, - { url = "https://files.pythonhosted.org/packages/0b/64/93232511a7906a492b1b7dfdfc17f4e95982d76a24ef4f86d18cfe7ae2c9/pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746", size = 6135280 }, - { url = "https://files.pythonhosted.org/packages/10/f2/b550b1f65cc7e51c9116b220b50aade60c439103432a3fd5b12efbc77e15/pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8", size = 4880030 }, - { url = "https://files.pythonhosted.org/packages/fe/4b/2f8f6f94643b9fe2083338eff294feda84d916409b5840b7a402d2be93f8/pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf", size = 6184439 }, - { url = "https://files.pythonhosted.org/packages/19/9b/c57569132174786aa3f72275ac306956859a639dad0ce8d95c8411ce8209/pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d", size = 8660747 }, - { url = "https://files.pythonhosted.org/packages/0e/ab/1c2159ec757677c5a6b8803f6be45c2b550dc42c84ec4a228dc219849bbb/pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb", size = 5626805 }, - { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779 }, +sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524 }, + { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102 }, + { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667 }, + { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185 }, + { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103 }, + { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391 }, + { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997 }, + { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581 }, + { url = "https://files.pythonhosted.org/packages/ab/0d/63670fc527e664068b70b7cab599aa38b7420dd009bdc29ea257e7f3dfb3/pyproj-3.7.1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a94e26c1a4950cea40116775588a2ca7cf56f1f434ff54ee35a84718f3841a3d", size = 6264315 }, + { url = "https://files.pythonhosted.org/packages/25/9d/cbaf82cfb290d1f1fa42feb9ba9464013bb3891e40c4199f8072112e4589/pyproj-3.7.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:263b54ba5004b6b957d55757d846fc5081bc02980caa0279c4fc95fa0fff6067", size = 4666267 }, + { url = "https://files.pythonhosted.org/packages/79/53/24f9f9b8918c0550f3ff49ad5de4cf3f0688c9f91ff191476db8979146fe/pyproj-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d6a2ccd5607cd15ef990c51e6f2dd27ec0a741e72069c387088bba3aab60fa", size = 9680510 }, + { url = "https://files.pythonhosted.org/packages/3c/ac/12fab74a908d40b63174dc704587febd0729414804bbfd873cabe504ff2d/pyproj-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5dcf24ede53d8abab7d8a77f69ff1936c6a8843ef4fcc574646e4be66e5739", size = 9493619 }, + { url = "https://files.pythonhosted.org/packages/c4/45/26311d6437135da2153a178125db5dfb6abce831ce04d10ec207eabac70a/pyproj-3.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c2e7449840a44ce860d8bea2c6c1c4bc63fa07cba801dcce581d14dcb031a02", size = 10709755 }, + { url = "https://files.pythonhosted.org/packages/99/52/4ecd0986f27d0e6c8ee3a7bc5c63da15acd30ac23034f871325b297e61fd/pyproj-3.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0829865c1d3a3543f918b3919dc601eea572d6091c0dd175e1a054db9c109274", size = 10642970 }, + { url = "https://files.pythonhosted.org/packages/3f/a5/d3bfc018fc92195a000d1d28acc1f3f1df15ff9f09ece68f45a2636c0134/pyproj-3.7.1-cp311-cp311-win32.whl", hash = "sha256:6181960b4b812e82e588407fe5c9c68ada267c3b084db078f248db5d7f45d18a", size = 5868295 }, + { url = "https://files.pythonhosted.org/packages/92/39/ef6f06a5b223dbea308cfcbb7a0f72e7b506aef1850e061b2c73b0818715/pyproj-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ad0ff443a785d84e2b380869fdd82e6bfc11eba6057d25b4409a9bbfa867970", size = 6279871 }, + { url = "https://files.pythonhosted.org/packages/e6/c9/876d4345b8d17f37ac59ebd39f8fa52fc6a6a9891a420f72d050edb6b899/pyproj-3.7.1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:2781029d90df7f8d431e29562a3f2d8eafdf233c4010d6fc0381858dc7373217", size = 6264087 }, + { url = "https://files.pythonhosted.org/packages/ff/e6/5f8691f8c90e7f402cc80a6276eb19d2ec1faa150d5ae2dd9c7b0a254da8/pyproj-3.7.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d61bf8ab04c73c1da08eedaf21a103b72fa5b0a9b854762905f65ff8b375d394", size = 4669628 }, + { url = "https://files.pythonhosted.org/packages/42/ec/16475bbb79c1c68845c0a0d9c60c4fb31e61b8a2a20bc18b1a81e81c7f68/pyproj-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04abc517a8555d1b05fcee768db3280143fe42ec39fdd926a2feef31631a1f2f", size = 9721415 }, + { url = "https://files.pythonhosted.org/packages/b3/a3/448f05b15e318bd6bea9a32cfaf11e886c4ae61fa3eee6e09ed5c3b74bb2/pyproj-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084c0a475688f934d386c2ab3b6ce03398a473cd48adfda70d9ab8f87f2394a0", size = 9556447 }, + { url = "https://files.pythonhosted.org/packages/6a/ae/bd15fe8d8bd914ead6d60bca7f895a4e6f8ef7e3928295134ff9a7dad14c/pyproj-3.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a20727a23b1e49c7dc7fe3c3df8e56a8a7acdade80ac2f5cca29d7ca5564c145", size = 10758317 }, + { url = "https://files.pythonhosted.org/packages/9d/d9/5ccefb8bca925f44256b188a91c31238cae29ab6ee7f53661ecc04616146/pyproj-3.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bf84d766646f1ebd706d883755df4370aaf02b48187cedaa7e4239f16bc8213d", size = 10771259 }, + { url = "https://files.pythonhosted.org/packages/2a/7d/31dedff9c35fa703162f922eeb0baa6c44a3288469a5fd88d209e2892f9e/pyproj-3.7.1-cp312-cp312-win32.whl", hash = "sha256:5f0da2711364d7cb9f115b52289d4a9b61e8bca0da57f44a3a9d6fc9bdeb7274", size = 5859914 }, + { url = "https://files.pythonhosted.org/packages/3e/47/c6ab03d6564a7c937590cff81a2742b5990f096cce7c1a622d325be340ee/pyproj-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:aee664a9d806612af30a19dba49e55a7a78ebfec3e9d198f6a6176e1d140ec98", size = 6273196 }, + { url = "https://files.pythonhosted.org/packages/ef/01/984828464c9960036c602753fc0f21f24f0aa9043c18fa3f2f2b66a86340/pyproj-3.7.1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:5f8d02ef4431dee414d1753d13fa82a21a2f61494737b5f642ea668d76164d6d", size = 6253062 }, + { url = "https://files.pythonhosted.org/packages/68/65/6ecdcdc829811a2c160cdfe2f068a009fc572fd4349664f758ccb0853a7c/pyproj-3.7.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0b853ae99bda66cbe24b4ccfe26d70601d84375940a47f553413d9df570065e0", size = 4660548 }, + { url = "https://files.pythonhosted.org/packages/67/da/dda94c4490803679230ba4c17a12f151b307a0d58e8110820405ca2d98db/pyproj-3.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83db380c52087f9e9bdd8a527943b2e7324f275881125e39475c4f9277bdeec4", size = 9662464 }, + { url = "https://files.pythonhosted.org/packages/6f/57/f61b7d22c91ae1d12ee00ac4c0038714e774ebcd851b9133e5f4f930dd40/pyproj-3.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b35ed213892e211a3ce2bea002aa1183e1a2a9b79e51bb3c6b15549a831ae528", size = 9497461 }, + { url = "https://files.pythonhosted.org/packages/b7/f6/932128236f79d2ac7d39fe1a19667fdf7155d9a81d31fb9472a7a497790f/pyproj-3.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8b15b0463d1303bab113d1a6af2860a0d79013c3a66fcc5475ce26ef717fd4f", size = 10708869 }, + { url = "https://files.pythonhosted.org/packages/1d/0d/07ac7712994454a254c383c0d08aff9916a2851e6512d59da8dc369b1b02/pyproj-3.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:87229e42b75e89f4dad6459200f92988c5998dfb093c7c631fb48524c86cd5dc", size = 10729260 }, + { url = "https://files.pythonhosted.org/packages/b0/d0/9c604bc72c37ba69b867b6df724d6a5af6789e8c375022c952f65b2af558/pyproj-3.7.1-cp313-cp313-win32.whl", hash = "sha256:d666c3a3faaf3b1d7fc4a544059c4eab9d06f84a604b070b7aa2f318e227798e", size = 5855462 }, + { url = "https://files.pythonhosted.org/packages/98/df/68a2b7f5fb6400c64aad82d72bcc4bc531775e62eedff993a77c780defd0/pyproj-3.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3caac7473be22b6d6e102dde6c46de73b96bc98334e577dfaee9886f102ea2e", size = 6266573 }, ] [[package]] @@ -2952,6 +3025,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a", size = 93002 }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + [[package]] name = "soupsieve" version = "2.6" @@ -3151,6 +3233,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "tblib" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/95/4b3044ec4bf248186769629bbfb495a458deb6e4c1f9eff7f298ae1e336e/tblib-3.1.0.tar.gz", hash = "sha256:06404c2c9f07f66fee2d7d6ad43accc46f9c3361714d9b8426e7f47e595cd652", size = 30766 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/44/aa5c8b10b2cce7a053018e0d132bd58e27527a0243c4985383d5b6fd93e9/tblib-3.1.0-py3-none-any.whl", hash = "sha256:670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e", size = 12552 }, +] + [[package]] name = "terminado" version = "0.18.1" @@ -3347,16 +3438,25 @@ wheels = [ [[package]] name = "xarray" -version = "2024.9.0" +version = "2025.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "packaging" }, { name = "pandas" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/d3/ae7a92c8448c40cd43f97fff93b1a57f87565b412fdc02eb14af5d4c3823/xarray-2024.9.0.tar.gz", hash = "sha256:e796a6b3eaec11da24f33e4bb14af41897011660a0516fa4037d3ae4bbd1d378", size = 3747432 } +sdist = { url = "https://files.pythonhosted.org/packages/9b/29/37761364e137db13898cf5a790574dd7883f7355d5dfb42b66ee7a9a6318/xarray-2025.4.0.tar.gz", hash = "sha256:2a89cd6a1dfd589aa90ac45f4e483246f31fc641836db45dd2790bb78bd333dc", size = 2974151 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/1e/96fd96419fec1a37da998a1ca3d558f2cae2f6f3cd5015170371b05a2b6b/xarray-2025.4.0-py3-none-any.whl", hash = "sha256:b27defd082c5cb85d32c695708de6bb05c2838fb7caaf3f952982e602a35b9b8", size = 1290171 }, +] + +[[package]] +name = "zict" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/ac/3c494dd7ec5122cff8252c1a209b282c0867af029f805ae9befd73ae37eb/zict-3.0.0.tar.gz", hash = "sha256:e321e263b6a97aafc0790c3cfb3c04656b7066e6738c37fffcca95d803c9fba5", size = 33238 } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/28/3a6365e45721c7c9078968ed94b4a60076bc31d73b8519021a69b4995b63/xarray-2024.9.0-py3-none-any.whl", hash = "sha256:4fd534abdf12d5fa75dd566c56483d5081f77864462cf3d6ad53e13f9db48222", size = 1191607 }, + { url = "https://files.pythonhosted.org/packages/80/ab/11a76c1e2126084fde2639514f24e6111b789b0bfa4fc6264a8975c7e1f1/zict-3.0.0-py2.py3-none-any.whl", hash = "sha256:5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae", size = 43332 }, ] [[package]] From 631aac5db6171ca765d788f444350d40fdd2b00b Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 6 Jun 2025 11:18:42 -0700 Subject: [PATCH 46/54] fix: update model and dataset --- src/geodata/datasets/__init__.py | 48 +++++++++++++++++++++++++++- src/geodata/datasets/_base.py | 17 ++++++++-- src/geodata/model/wind/_base.py | 54 ++++++++++++++++++++++++-------- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index f6c7defc..4d5955fe 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -26,7 +26,53 @@ from ._base import DatasetType from ._base import _registry as registry -__all__ = ["era5", "merra2", "registry", "register_hrrr", "DatasetType"] +__all__ = [ + "era5", + "merra2", + "registry", + "register_hrrr", + "DatasetType", + "load_dataset", + "list_datasets", +] + + +def load_dataset(weather_data_config: str) -> DatasetType: + """Load a dataset based on the provided weather data configuration. + This function retrieves the dataset class from the registry based on the provided configuration string. + It is very similar to the `get` method of the registry (e.g. `registry.get(weather_data_config)`), + but it is more explicit in its intent to load a dataset. + + Args: + weather_data_config (str): The configuration string for the dataset. + + Returns: + DatasetType: An instance of the dataset class corresponding to the configuration. + + Raises: + ValueError: If the provided configuration is not registered in the registry. + + """ + if weather_data_config not in registry: + raise ValueError(f"Dataset '{weather_data_config}' is not registered.") + + return registry[weather_data_config] + + +def list_datasets() -> list[str]: + """List all registered datasets. + + Returns: + list[str]: A list of dataset names that are registered in the registry. + + Raises: + ValueError: If no datasets are registered in the registry. + """ + + if not registry: + raise ValueError("No datasets are registered in the registry.") + + return list(registry.keys()) def register_hrrr(): diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 4deed3a7..8ac18bd2 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -335,14 +335,26 @@ def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): return ds def trim_variables( - self, variables: Sequence[str] | None = None, **kwargs + self, + ds: xr.Dataset | xr.DataArray, + variables: Sequence[str] | None = None, + **kwargs, ) -> xr.Dataset | xr.DataArray: """Method to trim the dataset to only include the specified variables. Args: + ds: The dataset to trim. variables: A sequence of strings representing the variables to keep. If None, we will keep the variables specified in the `variables` attribute of the dataset. + + Returns: + xr.Dataset | xr.DataArray: The trimmed dataset containing only the + specified variables. + + Raises: + ValueError: If the dataset does not have a `variables` attribute + defined and no variables are specified. """ if variables is None: @@ -353,7 +365,7 @@ def trim_variables( ) variables: Sequence[str] = getattr(self, "variables") - return kwargs["ds"][variables] + return ds[variables] def __repr__(self): return "".format( @@ -415,7 +427,6 @@ def tasks_func( xs: CoordRange, ys: CoordRange, yearmonths: xr.DataArray, - prepare_func, **meta_attrs, ): """A method that returns a list of tasks that can be run on the dataset.""" diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index de7920c4..244f81e3 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -60,14 +60,53 @@ class WindBaseModel(BaseModel): type: str = "wind" - def estimate_power( + def estimate( + self, + years: slice | None = None, + months: slice | None = None, + xs: slice | None = None, + ys: slice | None = None, + **kwargs, + ) -> xr.DataArray: + """Estimate wind speed or CF at the given locations and times. If a turbine is + specified, the CF is calculated based on the wind speed and the turbine's power curve. + Otherwise, pass in the `height` keyword argument to estimate wind speed at a specific height. + + Args: + years (slice, optional): Years. + months (slice, optional): Months. If None, all months are estimated. + xs (slice, optional): X coordinates. If None, all x coordinates in source are estimated. + ys (slice, optional): Y coordinates. If None, all y coordinates in source are estimated. + **kwargs: Additional keyword arguments to pass to the model. + - `turbine` (str): Name of the wind turbine to estimate power output. + - `height` (int): Height at which to estimate wind speed. If not specified, the model will use the default height. + + Raises: + ValueError: If neither 'turbine' nor 'height' is specified in kwargs. + + Returns: + xr.DataArray: Estimated wind speed. + """ + + if "turbine" in kwargs: + return self._estimate_power( + years=years, months=months, xs=xs, ys=ys, **kwargs + ) + + if "height" not in kwargs: + raise ValueError( + "Either 'turbine' or 'height' must be specified to estimate wind speed." + ) + + return super().estimate(years, months, xs, ys, **kwargs) + + def _estimate_power( self, turbine: str, xs: slice | None = None, ys: slice | None = None, years: slice | None = None, months: slice | None = None, - include_raw_power: bool = False, ) -> None: """Estimate wind speed at the given locations and times. @@ -77,7 +116,6 @@ def estimate_power( ys (slice, optional): Y slice. Defaults to None. years (slice, optional): Year slice. Defaults to None. months (slice, optional): Month slice. Defaults to None. - include_raw_power (bool, optional): Include raw power output. Defaults to False. Returns: xr.DataArray: Estimated wind speed. @@ -109,14 +147,4 @@ def estimate_power( output_dtypes=[float], ) - if include_raw_power: - # Calculate the capacity factor - cf: xr.DataArray = power / turbineconf["P"] - cf.attrs["units"] = "dimensionless" - cf.attrs["long_name"] = "Capacity factor" - cf.attrs["description"] = "Capacity factor of the wind turbine" - cf.attrs["turbine"] = turbine - - return xr.Dataset({"power": power, "cf": cf}) - return xr.Dataset({"cf": power / turbineconf["P"]}) From 6f6006442315b9065eb85574a067a47310cff056 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Fri, 6 Jun 2025 11:51:39 -0700 Subject: [PATCH 47/54] fix: herbie now works on cluster --- src/geodata/datasets/hrrr/hourly/wind_3d.py | 71 ++++++--------------- 1 file changed, 20 insertions(+), 51 deletions(-) diff --git a/src/geodata/datasets/hrrr/hourly/wind_3d.py b/src/geodata/datasets/hrrr/hourly/wind_3d.py index a043e375..734856f9 100644 --- a/src/geodata/datasets/hrrr/hourly/wind_3d.py +++ b/src/geodata/datasets/hrrr/hourly/wind_3d.py @@ -19,7 +19,7 @@ import pandas as pd import xarray as xr -from herbie import FastHerbie, Herbie +from herbie import FastHerbie from ....logging import redirect_stdout_to_logger from ..._base import AtomicDataset @@ -58,68 +58,35 @@ def _download_file(self, file: AtomicDataset): inclusive="left", ) - fh = FastHerbie( - date_range, - model=self.module, - product=self.product, - max_threads=mp.cpu_count() * 2, - save_dir=self._herbie_save_dir.name, - priority=self._priority, - ) - with redirect_stdout_to_logger(logger, logging.INFO): logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + fh.download(":[UV]GRD:[1,8]0 m") fh.download(":[UV]GRD:[1234] hybrid level") fh.download(":HGT:[1234] hybrid level") - uv_10 = [] - uv_80 = [] - uv_hybrid = [] - hgt_hybrid = [] - - for hour in date_range: - h = Herbie( - hour, - model=self.module, - product=self.product, - save_dir=self._herbie_save_dir.name, - priority=self._priority, - ) - - try: - uv_10.append( - h.xarray(":[UV]GRD:10 m", remove_grib=False).rename( - {"u10": "u", "v10": "v"} - ) - ) - uv_80.append(h.xarray(":[UV]GRD:80 m", remove_grib=False)) - uv_hybrid.append( - h.xarray(":[UV]GRD:[1234] hybrid level", remove_grib=False) - ) - hgt_hybrid.append( - h.xarray(":HGT:[1234] hybrid level", remove_grib=False) - ) - except ValueError: - logger.warning(f"No data found for {hour}, skipping.") - - # Offload to disk temporarily to save memory - xr.concat(uv_10, dim="time").to_netcdf( - os.path.join(self._herbie_save_dir.name, "uv_10.nc") - ) - del uv_10 - xr.concat(uv_80, dim="time").to_netcdf( + fh.xarray(":[UV]GRD:10 m", remove_grib=False).rename( + {"u10": "u", "v10": "v"} + ).to_netcdf(os.path.join(self._herbie_save_dir.name, "uv_10.nc")) + + fh.xarray(":[UV]GRD:80 m", remove_grib=False).to_netcdf( os.path.join(self._herbie_save_dir.name, "uv_80.nc") ) - del uv_80 - xr.concat(uv_hybrid, dim="time").to_netcdf( + fh.xarray(":[UV]GRD:[1234] hybrid level", remove_grib=False).to_netcdf( os.path.join(self._herbie_save_dir.name, "uv_hybrid.nc") ) - del uv_hybrid - xr.concat(hgt_hybrid, dim="time").to_netcdf( + fh.xarray(":HGT:[1234] hybrid level", remove_grib=False).to_netcdf( os.path.join(self._herbie_save_dir.name, "hgt_hybrid.nc") ) - del hgt_hybrid uv_10 = xr.open_dataset( os.path.join(self._herbie_save_dir.name, "uv_10.nc"), chunks="auto" @@ -162,6 +129,8 @@ def _download_file(self, file: AtomicDataset): del ds.attrs["search"] del ds.attrs["local_grib"] del ds.attrs["remote_grib"] + del ds["lon"] + del ds["lat"] except KeyError: pass From cd9a52196ba3445f5a97ed9d51590ce3e916774d Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Sat, 7 Jun 2025 22:07:36 -0700 Subject: [PATCH 48/54] fix: HRRR edge case with last day of the month --- src/geodata/datasets/hrrr/hourly/wind_3d.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/geodata/datasets/hrrr/hourly/wind_3d.py b/src/geodata/datasets/hrrr/hourly/wind_3d.py index 734856f9..a70441ff 100644 --- a/src/geodata/datasets/hrrr/hourly/wind_3d.py +++ b/src/geodata/datasets/hrrr/hourly/wind_3d.py @@ -16,6 +16,7 @@ import logging import multiprocessing as mp import os +from datetime import datetime, timedelta import pandas as pd import xarray as xr @@ -51,12 +52,9 @@ class HRRR3DWindHourlyDataset(HRRRBaseDataset): def _download_file(self, file: AtomicDataset): year, month, day = file.year, file.month, file.day - date_range = pd.date_range( - f"{year}-{month}-{day}", - f"{year}-{month}-{day+1}", - freq="h", - inclusive="left", - ) + start = datetime(year, month, day) + end = start + timedelta(days=1) + date_range = pd.date_range(start, end, freq="h", inclusive="left") with redirect_stdout_to_logger(logger, logging.INFO): logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") From f2b0b055302d7b3612afb58645aa3c45fb35e9e9 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 9 Jun 2025 12:18:34 -0700 Subject: [PATCH 49/54] docs: update dataset overview and remove outdated pages --- docs/source/conf.py | 2 +- docs/source/datasets/era5.rst | 81 +++ docs/source/datasets/era5/era5.ipynb | 580 --------------------- docs/source/datasets/era5/era5_download.md | 217 -------- docs/source/datasets/era5/era5_outputs.md | 152 ------ docs/source/datasets/era5/index.md | 58 --- docs/source/datasets/merra2/index.md | 15 +- docs/source/datasets/overview.rst | 6 +- 8 files changed, 93 insertions(+), 1018 deletions(-) create mode 100644 docs/source/datasets/era5.rst delete mode 100644 docs/source/datasets/era5/era5.ipynb delete mode 100644 docs/source/datasets/era5/era5_download.md delete mode 100644 docs/source/datasets/era5/era5_outputs.md delete mode 100644 docs/source/datasets/era5/index.md diff --git a/docs/source/conf.py b/docs/source/conf.py index ccd14f5d..ea9d69e7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ from geodata import __version__ project = "Geodata" -copyright = "2023, Geodata Contributors" +copyright = "2025, Geodata Contributors" author = "Geodata Contributors" release = __version__ diff --git a/docs/source/datasets/era5.rst b/docs/source/datasets/era5.rst new file mode 100644 index 00000000..2af0aee6 --- /dev/null +++ b/docs/source/datasets/era5.rst @@ -0,0 +1,81 @@ +ERA5 Specific Instructions +========================== + +This page explains how you can set up access to ERA5 data from the `Copernicus Data Store `_. + +Creating a CDS account +---------------------- + +To download ERA5 data from the CDS, you'll need to create a free `CDS account here `_. + +Download data through CDS API +----------------------------- + +Once your account has been created, set up access to the API by following these steps: + +1. Log into your CDS account and visit your `profile page `_. +2. Install the API key. There will be a section called **Personal Access Token**. + Copy these two lines into a file called ``.cdsapirc`` in your user root folder. + +- **macOS/Linux**: Open a terminal and run: + + .. code-block:: bash + + touch ~/.cdsapirc + + Then add the lines using: + + .. code-block:: bash + + echo [line 1 of the code] >> ~/.cdsapirc + echo [line 2 of the code] >> ~/.cdsapirc + + + - **Windows**: The process is slightly more complicated. Please refer to the in-depth guide at the Copernicus Knowledge Base `here `_. + +3. Install the CDS API client by opening a terminal/shell and running + +.. code-block:: bash + + pip install ".[download]" + +(Assuming you are in Geodata's *root directory*.) + +1. Once you've installed the API key and the API client, confirm access by running an + example in a Python script or a Jupyter notebook: + +.. code-block:: python + + import cdsapi + + c = cdsapi.Client() + + c.retrieve( + "reanalysis-era5-single-levels", + { + "product_type": "reanalysis", + "format": "netcdf", + "variable": [ + "2m_dewpoint_temperature", + "2m_temperature", + ], + "year": "2011", + "month": [ + "01", + ], + "day": ["01", "02", "03"], + "time": [ + "00:00", + "12:00", + ], + }, + "download.nc", + ) + +The above example downloads 2m temperature and 2m dewpoint temperature with data points +at 00:00 and 12:00 for each day, from January 1-3, 2011, in NetCDF format. + +If this works, you have successfully set up access to the ERA5 data through the CDS API. +Please subsequently refer to the `general documentation on datasets <../overview.rst>`_ +for more information on how to download ERA5-based datasets using the ``geodata`` +package. diff --git a/docs/source/datasets/era5/era5.ipynb b/docs/source/datasets/era5/era5.ipynb deleted file mode 100644 index 7d8d0696..00000000 --- a/docs/source/datasets/era5/era5.ipynb +++ /dev/null @@ -1,580 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ERA5 Analysis Process\n", - "\n", - "This Jupyter notebook provides a brief overview of how to use the **geodata** package to download ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview), create geographic-temporal subsets called cutouts, and use those cutouts to generate standalone datasets for separate analysis.\n", - "\n", - "*The following guide assumes you have installed and configured **geodata** and all required dependencies.*" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1 - Setup\n", - "\n", - "Import the package first." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import geodata" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notifications in **geodata** are implemented using `loggers` from the `logging` library.\n", - "It is recommended to always launch a logger to get information on what is going on. For debugging, you can use the more verbose `level=logging.DEBUG`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "logging.basicConfig(level=logging.INFO)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2 - Download and Create Cutout\n", - "Assuming you have previously created a CDS account and set up the CDS API credentials, you can download ERA5 data from the CDS API as follows.\n", - "\n", - "First, define a dataset object for the data you wish to download:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For ERA5, pass geographic bounds in array as follows:\n", - "## bounds = [North, West, South, East]\n", - "## Omitting bounds will default to global file of 20+ GB per month\n", - "DS = geodata.Dataset(\n", - " module=\"era5\",\n", - " weather_data_config=\"wind_solar_hourly\",\n", - " years=slice(2005, 2005),\n", - " months=slice(1, 2),\n", - " bounds=[50, -3, 45, 3],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* Use `module` to specify the data source. In this example, it is \"era5\".\n", - "* Use `weather_data_config` to specifiy the dataset. In this example, hourly data is used, as specified by the `\"wind_solar_hourly\"` value.\n", - "* Use `years=slice()` and `months=slice()` to specify the years and months for download. In each parameter, the first value indicates the start period, and the second value the end period.\n", - "* Use `bounds` to specify the geographic bounds to which you wish to limit your download data. `bounds` should be set as follows: `bounds = [North, West, South, East]`. Omitting bounds will default to downloading a global file of 20+ GB per month.\n", - "\n", - "Use the code block below to begin the download." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When a `dataset` object is created, **geodata** performs a check to see if the data specified has already been downloaded by checking for the existence of ERA5 datafiles in the `era5` directory configured in `src/geodata/config.py` (downloaded data is placed into subdirectories by year and then - for daily files - by month, ie `2011/01, 2011/02, 2012/01`, etc). Monthly files are simply placed in the month's folder. If downloaded data is found, the `prepared` attribute is set to `True` upon `dataset` object declaration.\n", - "\n", - "Accordingly, the snippet below saves you the trouble of accidentally redownloading data if it is already present in the correct subdirectories." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if DS.prepared == False:\n", - " DS.get_data()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, in order to use the downloaded ERA5 data with **geodata**, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "DS.trim_variables()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`trim_variables()` subsets and resaves the downloaded files so that only those variables needed to generate **geodata** outputs are kept." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3 - Create Cutout\n", - "\n", - "A cutout is a subset of downloaded data based on specified time periods and geographic coordinates. Cutouts are saved to the cutout directory specified in `src/geodata/config.py` and can be used to generate multiple outputs.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout = geodata.Cutout(\n", - " name=\"era5-europe-test-2011-02\",\n", - " module=\"era5\",\n", - " weather_data_config=\"wind_solar_hourly\",\n", - " xs=slice(1, 2),\n", - " ys=slice(48, 46),\n", - " years=slice(2005, 2005),\n", - " months=slice(1, 1),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The above code creates a cutout for January 2011 for a geographic area corresponding to a portion of Europe. Walking through the parameters:\n", - "\n", - "* `name` will be the name of the directory created in the cutouts folder where **geodata** will place the data files corresponding to the cutout.\n", - "* `module` indicates the source for the data from which the cutout is created.\n", - "* Use `xs=slice()` and `ys=slice()` to define a geographical range for the cutout. These para\n", - "* Use `years=slice()` and `months=slice()` to define a temporal range for the cutout. \n", - "\n", - "`geodata.Cutout()` only defines the cutout object in memory. To actually create the cutout files, run `prepare()`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.prepare()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Running `cutout.prepare()` as above will create the cutout by downloading and then subsetting the ERA5 data. Accordingly, the above code block could take a while to finish processing.\n", - "\n", - "`prepare()` will first perform a check to see if a cutout has already been created at the specified directory, and will exit the download. creation process if a cutout already exists. To override this behavior and force a redownload and recalculation of the cutout, run `prepare(overwrite=True)`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To verify the results of the cutout, you can print some attributes to the console as follows.\n", - "\n", - "Basic information:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Name:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Coordinates:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.coords" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "All metadata:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.meta" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4 - Generate Outputs\n", - "\n", - "**geodata** currently supports the following outputs using ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview).\n", - "\n", - "### Wind\n", - "* Wind generation time-series (`wind`)\n", - "* Wind speed time-series (`windspd`)\n", - "\n", - "### Solar\n", - "* Solar photovoltaic generation time-series (`pv`)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Wind Generation Time-series\n", - "Convert wind speeds for turbine to wind energy generation using the following code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_wind = geodata.convert.wind(cutout, turbine=\"Suzlon_S82_1.5_MW\", smooth=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `turbine` - **string or dict** - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of Turbines here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/windturbine)\n", - "* `smooth` - **bool or dict** - If True smooth power curve with a gaussian kernel as determined for the Danish wind fleet to Delta_v = 1.27 and sigma = 2.29. A dict allows to tune these values.\n", - "\n", - "*Note* - \n", - "You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_wind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind = ds_wind.to_dataframe(name=\"wind\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind.to_csv(\"era5_wind_data.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Wind Speed Density Time-series\n", - "Extract wind speeds at given height (ms-1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_windspd = geodata.convert.windspd(cutout, turbine=\"Vestas_V66_1750kW\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `**params` - Must have 1 of the following:\n", - " - `turbine` - **string or dict** - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of Turbines here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/windturbine)\n", - " - `hub-height` - **num** - Extrapolation height (m)\n", - " \n", - "*Note* - \n", - "You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_windspd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd = ds_windspd.to_dataframe(name=\"windspd\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd.to_csv(\"era_windspd_data.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Solar Photovoltaic Generation Time-series\n", - "\n", - "Convert downward-shortwave, upward-shortwave radiation flux and ambient temperature into a pv generation time-series.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_pv = geodata.convert.pv(cutout, panel=\"KANEKA\", orientation=\"latitude_optimal\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel)\n", - "* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions.\n", - "* (optional) clearsky_model - string or None - \tEither the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_pv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv = ds_pv.to_dataframe(name=\"pv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv.to_csv(\"era_pv_data.csv\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/source/datasets/era5/era5_download.md b/docs/source/datasets/era5/era5_download.md deleted file mode 100644 index 9f71c48a..00000000 --- a/docs/source/datasets/era5/era5_download.md +++ /dev/null @@ -1,217 +0,0 @@ -# Download ERA5 Dataset and Create ERA5 Cutouts - -A short guide to using **geodata** to downloading and creating cutouts from ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/). - -## Download Dataset - -Download methods for ERA5 data are built into the **geodata** package. A basic example is as follows: - - -To start, import the required dependencies: - -```python -import geodata - -dataset = geodata.Dataset( - module="era5", - weather_data_config="wind_solar_hourly", - years=slice(2005, 2005), - months=slice(1, 2), - bounds=[50, -3, 45, 3], -) - -if not dataset.prepared: - dataset.get_data() - -dataset.trim_variables(downloadedfiles=True) -``` - -Let's breakdown the code above: - -```python -import geodata -``` -Importing the logging package allows **geodata** to generate console input for download status, errors, etc. -Importing geodata is necessary to run **geodata**. - -```python -dataset = geodata.Dataset( - module="era5", - weather_data_config="wind_solar_hourly", - years=slice(2005, 2005), - months=slice(1, 2), - bounds=[50, -3, 45, 3], -) -``` - -`geodata.Dataset()` creates a dataset object via which you can download data files. To create objects for ERA5 data, specify `module="era5"`. You must also have configured `era5_dir` in `config.py` to point to a directory on your local machine (to set up `config.py`, [see here](../../quick_start/packagesetup.md)). - -The `years` and `months` parameters allow you to specify start years/months and end years/months for the data download. The above example would download data for January to February 2005. Ranges based on more granular time periods (such as day or hour) are not currently supported, but may be available in a future release. - -Running the above code does not actually download the data yet. Instead, it checks whether the indicated files for download are present in the local directory specified in `config.py`: - -``` ->> INFO:geodata.dataset:Directory /home/user/.local/geodata/era5 found, checking for completeness. ->> INFO:geodata.dataset:Directory complete. -``` - -and returns a `dataset` object indicating whether the data is "prepared." - -``` - -``` - -A "prepared" dataset indicates that the directories for storing the data - which take the form `/era5/{years}/{months}` for every unique time period in the data - have been created and are populated with downloaded data. - -```python -if not dataset.prepared: - dataset.get_data() -``` -If the dataset is "Unprepared", this conditional statement will download the actual netcdf files from the ERA5 CDS API. - - -```python -dataset.trim_variables(downloadedfiles = True) -``` -To save hard disk space, **geodata** allows you to trim the downloaded datasets to just the variables needed for the package's supported outputs. - -## Create Cutout - -A cutout is the basis for any data or analysis output by the **geodata** package. Cutouts are stored in the directory `cutout_dir` configured in `config.py` (to set up `config.py`, [see here](../../quick_start/packagesetup.md)). - -```python -cutout = geodata.Cutout( - name="era5-europe-test-2011-01", - module="era5", - weather_data_config="era5_monthly", - xs=slice(30, 41.56244222), - ys=slice(33.56459975, 35), - years=slice(2011, 2011), - months=slice(1, 1), -) -``` - -To prepare a cutout, the following must be specified for `geodata.Cutout()`: - -* The cutout name -* The source dataset -* Time range -* Geographic range as represented by `x` and `y` coordinates. - -The example in the code block above uses ERA5 data, as specified by the `module=era5` parameter. - -`xs` and `ys` in combination with the `slice()` function allow us to specify a geographic range based on longitude and latitude. The above example subsets a portion of Europe. - -`years` and `months` are used to subset the time range. For both functions, the first value represents the start point, and the second value represents the end point. The above example creates a cutout for January 2011. - -## Prepare Cutout - -To create a cutout, we can run `cutout.prepare()`. The **geodata** package will create a folder in the cutout directory you specified in `config.py` with the name specified in `geodata.Cutout()` (in the above example, `era5-europe-test-2011-01`). The folder, depending on the date range, will then contain one or more monthly netcdf files containing ERA5 data corresponding to the temporal and geographical ranges indicated when the cutout was created. Data files in the cutout folder will be at the monthly level - i.e., there will be one file for each month in the specified download time range. - -To actually create the cutout, you must run `cutout.prepare()`. Upon running `cutout.prepare()`, **geodata** will check for the presence of the cutout and abort if the cutout already exists. If you want to force the regeneration of a cutout, run the command with the parameter `overwrite=True`. - - -## Cutout Metadata - -You can query various metadata associated with a cutout. Querying returns the name, geographic and time range, and the preparation status of the cutout (i.e., whether cutout.prepare() has been run, creating the .nc files making up the cutout data). - -```python - -``` - -- `cutout.name` returns just the name: - -``` -'era5-europe-test-2011-01' -``` - -- `cutout.coords` returns coordinates: -```python -Coordinates: - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 - lon (x) float32 ... - lat (y) float32 ... - * year-month (year-month) MultiIndex - - year (year-month) int64 2011 - - month (year-month) int64 1 -``` - -- `cutout.meta` returns all associated metadata: -```python - -Dimensions: (time: 744, x: 47, y: 5, year-month: 1) -Coordinates: - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 - lon (x) float32 ... - lat (y) float32 ... - * year-month (year-month) MultiIndex - - year (year-month) int64 2011 - - month (year-month) int64 1 -Data variables: - height (y, x) float32 ... -Attributes: - Conventions: CF-1.6 - history: 2020-03-14 04:29:35 GMT by grib_to_netcdf-2.16.0: /opt/ecmw... - module: era5 - view: {'x': slice(30, 41.56244222, None), 'y': slice(35, 33.56459... -``` - -To understand the variables that were downloaded, you can run: -``` -cutout.dataset_module.weather_data_config -``` - -Which will return the selected download settings for the ERA5 data. - - -```python -{'wind_solar_hourly': {'api_func': , - 'file_granularity': 'monthly', - 'tasks_func': , - 'meta_prepare_func': , - 'prepare_func': , - 'template': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_hourly.nc', - 'fn': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_hourly.nc', - 'product': 'reanalysis-era5-single-levels', - 'product_type': 'reanalysis', - 'variables': ['100m_u_component_of_wind', - '100m_v_component_of_wind', - '2m_temperature', - 'runoff', - 'soil_temperature_level_4', - 'surface_net_solar_radiation', - 'surface_pressure', - 'surface_solar_radiation_downwards', - 'toa_incident_solar_radiation', - 'total_sky_direct_solar_radiation_at_surface', - 'forecast_surface_roughness', - 'orography'], - 'meta_attrs': {'Conventions': 'CF-1.6', - 'history': '2022-01-11 00:43:10 GMT by grib_to_netcdf-2.23.0: /opt/ecmwf/mars-client/bin/grib_to_netcdf -S param -o /cache/data3/adaptor.mars.internal-1641861772.7584445-3425-17-8ff8bcdc-2c08-4aed-95db-e0337692a384.nc /cache/tmp/8ff8bcdc-2c08-4aed-95db-e0337692a384-adaptor.mars.internal-1641861207.3577378-3425-30-tmp.grib', - 'module': 'era5'}}, - 'wind_solar_monthly': {'api_func': , - 'file_granularity': 'monthly', - 'tasks_func': , - 'meta_prepare_func': , - 'prepare_func': , - 'template': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_monthly.nc', - 'fn': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_monthly.nc', - 'product': 'reanalysis-era5-single-levels-monthly-means', - 'product_type': 'monthly_averaged_reanalysis', - 'variables': ['100m_u_component_of_wind', - '100m_v_component_of_wind', - '2m_temperature', - 'runoff', - 'soil_temperature_level_4', - 'surface_net_solar_radiation', - 'surface_pressure', - 'surface_solar_radiation_downwards', - 'toa_incident_solar_radiation', - 'total_sky_direct_solar_radiation_at_surface', - 'forecast_surface_roughness', - 'orography']}} -``` diff --git a/docs/source/datasets/era5/era5_outputs.md b/docs/source/datasets/era5/era5_outputs.md deleted file mode 100644 index 5bf321e4..00000000 --- a/docs/source/datasets/era5/era5_outputs.md +++ /dev/null @@ -1,152 +0,0 @@ -# Generating Outputs with ERA5 Data - - - -**geodata** currently supports the following wind outputs using ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - -* Wind generation time-series (`wind`) -* Wind speed time-series (`windspd`) -* Solar photovoltaic generation time-series (`pv`) - -## Supported ERA5 Outputs - -### Wind Generation Time-series - -Convert wind speeds for turbine to wind energy generation. - -```python -cutout.wind(turbine: str | dict[str, str], smooth: bool = False) -``` - -#### Parameters - -* `cutout` - `Cutout` - A cutout created by `geodata.Cutout()` -* `turbine` - `str | dict` - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of turbines here.](https://github.com/GeodataTools/geodata/tree/master/src/geodata/resources/windturbine) -* `smooth` - `bool | dict` - If `True`, smooth power curve with a gaussian kernel as determined for the Danish wind fleet to Delta_v = 1.27 and sigma = 2.29. A dict allows to tune these values. - -*Note*: You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`). - -#### Example Code - -```python -ds_wind = cutout.wind(turbine="Suzlon_S82_1.5_MW", smooth=True) -ds_wind.to_dataframe(name="wind") -``` - -### Wind Speed Time-series - -Extract wind speeds at given height (ms-1) - - -``` -geodata.convert.windspd(cutout: geodata.Cutout, **params) -``` - -#### Parameters - -* `cutout` - `str` - A cutout created by `geodata.Cutout()` -* `**params` - Must have 1 of the following: - - `turbine` - `str | dict` - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, [the list of turbines here.](https://github.com/GeodataTools/geodata/tree/master/src/geodata/resources/windturbine) - - `hub-height` - `int` - Extrapolation height (m) -*Note*: You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`). -#### Example Code - -```python -ds_windspd = cutout.windspd(turbine='Vestas_V66_1750kW') -ds_windspd.to_dataframe(name = 'windspd') -``` - - -### Solar photovoltaic generation time-series - -Convert downward-shortwave, upward-shortwave radiation flux and ambient temperature into a pv generation time-series. - -```python -cutout.pv(panel, orientation, clearsky_model) -``` - -#### Parameters - -* `cutout` - **string** - A cutout created by `geodata.Cutout()` -* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel) -* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions. -* (optional) clearsky_model - string or None - Either the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity. - -#### Example Code and Result - -```python -ds_pv = geodata.convert.pv(panel="KANEKA", orientation = "latitude_optimal") -ds_pv.to_dataframe(name = 'pv') -``` - -## Example Output - -With the full list of supported ERA5 outputs above, we can see an example of generating output ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - - -### Setup - -Let's assume we've created an ERA5 cutout along the following lines: - -```python -cutout = geodata.Cutout(name="era5-europe-example", - module="era5", - weather_data_config="era5_monthly", - xs=slice(30, 41.56244222), - ys=slice(33.56459975, 35), - years=slice(2011, 2011), - months=slice(1,1) -) -``` - -Note that unlike datasets, we don't need explicitly prepare a cutout. Geodata will prepare the cutout automatically if it's not ready. We can now use this cutout to generate datasets. - -### Creating a Solar photovoltaic (pv) generation time-series - -To create a pv generation time-series, we can use the following code with our ERA5 cutout: - - -```python -ds = cutout.pv(panel="KANEKA", orientation="latitude_optimal") -``` - -Some information about the parameters: -* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel) -* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions. -* (optional) clearsky_model - string or None - Either the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity. - - -The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file: - -``` - -array([[[0., 0., 0., ..., 0., 0., 0.], - [0., 0., 0., ..., 0., 0., 0.], - [0., 0., 0., ..., 0., 0., 0.], - -Coordinates: - lon (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - lat (y) float32 34.815 34.565 34.315 34.065 33.815 - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 -``` - -To convert this array to a more conventional dataframe, we can run: - -```python -df = ds.to_dataframe(name='pv') -``` - -which converts the xarray dataset into a pandas dataframe. - - -The result is a dataset with an observation for each time period (in this ERA5 case, hourly) and geographic point with the calculated pv generation for each observation. - -Finally, we can run something like: - -```python -df.to_csv('era5_pv_data.csv') -``` - -to output the data to csv for use in other applications. diff --git a/docs/source/datasets/era5/index.md b/docs/source/datasets/era5/index.md deleted file mode 100644 index 462b826b..00000000 --- a/docs/source/datasets/era5/index.md +++ /dev/null @@ -1,58 +0,0 @@ -# ERA5 Related Tutorials - -This page explains how you can setup access to ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - -## Creating a CDS account - -To download the ERA5 data from the CDS, you'll need to create a free [CDS account here](https://cds.climate.copernicus.eu/). - -## Download data through CDS API -Once your account has been created, you'll need to set up access to the API by doing the following: - -1. Log into your CDS account and visit [your profile page](https://cds.climate.copernicus.eu/profile). -2. Install the API key. There will be a section called **Personal Access Token**. You will then need to copy these two lines this into - a file called _.cdsapirc_ that you will create in your user root folder. - - *macOS/Linux*: For macOS or Linux, you can create _.cdsapirc_ by opening a terminal window and running `touch ~/.cdsapirc` to create the file and then `echo [line 1 of the code] >> ~/.cdsapirc` followed by `echo [line 2 of the code] >> ~/.cdsapirc`. - - *Windows*: For Windows, the process is slightly more complicated. Please refer to an in-depth guide at the Copernicus Knowledge Base [here](https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+Windows). -3. Install the CDS API client by opening a terminal/shell and running `pip install ".[download]"`, assuming you are in Geodata's *root directory*. -4. Once you've installed the API key and the API client, you can confirm access by running an example in a Python script or a Jupyter notebook. - -```python -import cdsapi - -c = cdsapi.Client() - -c.retrieve( - "reanalysis-era5-single-levels", - { - "product_type": "reanalysis", - "format": "netcdf", - "variable": [ - "2m_dewpoint_temperature", - "2m_temperature", - ], - "year": "2011", - "month": [ - "01", - ], - "day": ["01", "02", "03"], - "time": [ - "00:00", - "12:00", - ], - }, - "download.nc", -) -``` - -The above example downloads 2m temperature and 2m dewpoint temperature with data points at 00:00 and 12:00 for each day, from January 1-3, 2011, in NetCDF format. - -## More Resources on ERA5 - -We've also provide a few tutorials on how to interact with the ERA5 datasets using Geodata. They can be found in the links below. - -```{toctree} -:maxdepth: 2 -:glob: -* -``` diff --git a/docs/source/datasets/merra2/index.md b/docs/source/datasets/merra2/index.md index 8e818eb3..fb020ff4 100644 --- a/docs/source/datasets/merra2/index.md +++ b/docs/source/datasets/merra2/index.md @@ -21,7 +21,7 @@ The following procedure is sourced from [GES DISC's documentation](https://disc. ## Configure API Crendentials -To download MERRA2 data via **geodata** you'll need to install the API credentials locally. +To download MERRA2 data via **geodata** you'll need to install the API credentials locally. ### macOS/Linux @@ -36,17 +36,14 @@ where `[login]` is your Earthdata user name and `[password]` is your Earthdata L ### Windows For Windows, open Notepad and enter the following line in a new document, making sure to substitute `` and ``for your Earthdata login credentials: - + `machine urs.earthdata.nasa.gov login password ` Save the file to `C:\Users\\.netrc` ## What' next? -We provide a few more tutorials on how to download and utilize the MERRA2 dataset with Geodata. You can find them below. - -```{toctree} -:maxdepth: 2 -:glob: -* -``` +Now that you have configured your Earthdata Login credentials, you have successfully set up access to the MERRA-2 data. +Please subsequently refer to the [general documentation on datasets](../overview.rst) +for more information on how to download ERA5-based datasets using the `geodata` +package. diff --git a/docs/source/datasets/overview.rst b/docs/source/datasets/overview.rst index d4ff5f64..57430059 100644 --- a/docs/source/datasets/overview.rst +++ b/docs/source/datasets/overview.rst @@ -11,7 +11,7 @@ Key Features - Supports the download and management of datasets from various sources, such as `ERA5 `_ and - `MERRA2 `. + `MERRA2 `_. - Provides a consistent API for accessing geospatial data, regardless of the underlying data source. @@ -87,3 +87,7 @@ Dataset's Interoperability with Cutout At the moment, the dataset classes are not interoperable with the `Cutout` class. In the future, we plan to consolidate the functionalities of the `Cutout` class into the dataset classes and the modeling module (see :doc:`here<../modeling/wind/index>`). + +For now, after downloading a dataset, a good point to move forward would be to use the +:doc:`modeling module <../modeling/index>` to create a model that can do certain types +of modeling with the dataset, such as wind speed estimation or solar PV generation. From 90c503dea7b09d258167edc56e69c3d658707379 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 9 Jun 2025 12:38:13 -0700 Subject: [PATCH 50/54] fix: update CI to conform with new dataset interface --- src/geodata/datasets/merra2/_base.py | 5 +- tests/pr/test_era5.py | 215 --------------------------- tests/pr/test_era5_lengthy.py | 42 +----- tests/pr/test_mask.py | 0 tests/pr/test_merra2.py | 157 +------------------ 5 files changed, 16 insertions(+), 403 deletions(-) delete mode 100644 tests/pr/test_era5.py delete mode 100644 tests/pr/test_mask.py diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index 194bc47d..bd99e79b 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -141,7 +141,7 @@ def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): # Only keep the variables that are needed for var in ds.data_vars: if var.lower() not in self.variables: - ds = ds.drop(var) + ds = ds.drop_vars(var) # Change all variables to lowercase return ds.rename({var: var.lower() for var in ds.data_vars}) @@ -214,7 +214,6 @@ def tasks_func( xs: CoordRange, ys: CoordRange, yearmonths: xr.DataArray, - prepare_func: callable, **meta_attrs, ): if not isinstance(xs, slice): @@ -236,7 +235,7 @@ def tasks_func( return [ dict( - prepare_func=prepare_func, + prepare_func=cls.prepare_func, xs=xs, ys=ys, year=year, diff --git a/tests/pr/test_era5.py b/tests/pr/test_era5.py deleted file mode 100644 index 7c4ad894..00000000 --- a/tests/pr/test_era5.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2022 Xiqiang Liu - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -from itertools import product - -import numpy as np -import xarray as xr - -import geodata - -logging.basicConfig(level=logging.INFO) - - -def get_data_configs() -> list[str]: - return ["wind_solar_hourly"] - - -def get_bounds() -> list[list[int]]: - return [[50, 0, 48, 3]] - - -def get_years() -> list[slice]: - return [slice(2005, 2005)] - - -def get_months() -> list[slice]: - return [slice(1, 1)] - - -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - -def get_turbine() -> str: - return "Suzlon_S82_1.5_MW" - - -def get_smooth() -> bool: - return True - - -def get_panel() -> str: - return "KANEKA" - - -def get_orientation() -> str: - return "latitude_optimal" - - -def get_era5(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="era5", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, - ) - if not dataset.prepared: - dataset.get_data(testing=True) - return dataset - - -def create_cutout(data_config: str, x: slice, y: slice, year: slice, month: slice): - cutout = geodata.Cutout( - name=f"era-{data_config}-{year.start}-{month.start}", - module="era5", - weather_data_config=data_config, - xs=x, - ys=y, - years=year, - months=month, - ) - cutout.prepare() - return cutout - - -def create_wind_output(cutout: geodata.Cutout, turbine, smooth): - ds_wind = cutout.wind(turbine, smooth) - df_wind = ds_wind.to_dataframe(name="wind") - return df_wind - - -def create_windspd_output(cutout: geodata.Cutout, turbine): - ds_windspd = cutout.wind(turbine) - df_windspd = ds_windspd.to_dataframe(name="windspd") - return df_windspd - - -def create_pv_output(cutout: geodata.Cutout, panel, orientation): - ds_pv = cutout.pv(panel, orientation) - df_pv = ds_pv.to_dataframe(name="pv") - return df_pv - - -def test_download(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in product(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] - - -def test_cutout(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - assert cutout.prepared - - -def test_wind_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "wind": np.floating, - } - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - smooth = get_smooth() - df_wind = create_wind_output(cutout, turbine, smooth) - - for col in supposed_dtypes: - assert np.issubdtype(df_wind[col].dtype, supposed_dtypes[col]) - - -def test_windspd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "windspd": np.floating, - } - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - df_windspd = create_windspd_output(cutout, turbine) - - for col in supposed_dtypes: - assert np.issubdtype(df_windspd[col].dtype, supposed_dtypes[col]) - - -def test_pv_output(): - configs = [config for config in get_data_configs() if "hourly" in config] - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "pv": np.floating, - } - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - panel = get_panel() - orientation = get_orientation() - df_pv = create_pv_output(cutout, panel, orientation) - - for col in supposed_dtypes: - assert np.issubdtype(df_pv[col].dtype, supposed_dtypes[col]) diff --git a/tests/pr/test_era5_lengthy.py b/tests/pr/test_era5_lengthy.py index 8fdcd37a..9979ace3 100644 --- a/tests/pr/test_era5_lengthy.py +++ b/tests/pr/test_era5_lengthy.py @@ -17,9 +17,7 @@ import logging -import geodata - -import xarray as xr +from geodata.datasets import DatasetType, load_dataset logging.basicConfig(level=logging.INFO) @@ -41,24 +39,13 @@ def get_months() -> list[slice]: return [slice(1, 2)] -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - def get_era5(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="era5", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, + dataset_cls = load_dataset(data_config) + dataset: DatasetType = dataset_cls( + years=year, months=month, bounds=bound, testing=True ) - if not dataset.prepared: - dataset.get_data() + if not dataset.downloaded: + dataset.download() return dataset @@ -70,19 +57,4 @@ def test_download(): for config, year, month, bound in zip(configs, years, months, bounds): dataset = get_era5(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] + assert dataset.downloaded diff --git a/tests/pr/test_mask.py b/tests/pr/test_mask.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/pr/test_merra2.py b/tests/pr/test_merra2.py index 27f8569a..ffff5169 100644 --- a/tests/pr/test_merra2.py +++ b/tests/pr/test_merra2.py @@ -15,10 +15,7 @@ import logging -import geodata - -import xarray as xr -import numpy as np +from geodata.datasets import DatasetType, load_dataset logging.basicConfig(level=logging.INFO) @@ -44,71 +41,16 @@ def get_months() -> list[slice]: return [slice(1, 1)] -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - -def get_turbine() -> str: - return "Suzlon_S82_1.5_MW" - - -def get_smooth() -> bool: - return True - - -def get_var_height() -> str: - return "lml" - - def get_merra2(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="merra2", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, + dataset_cls = load_dataset(data_config) + dataset: DatasetType = dataset_cls( + years=year, months=month, bounds=bound, testing=True ) - if not dataset.prepared: - dataset.get_data(testing=True) + if not dataset.downloaded: + dataset.download() return dataset -def create_cutout(data_config: str, x: slice, y: slice, year: slice, month: slice): - cutout = geodata.Cutout( - name=f"merra2-{data_config}-{year.start}-{month.start}", - module="merra2", - weather_data_config=data_config, - xs=x, - ys=y, - years=year, - months=month, - ) - cutout.prepare() - return cutout - - -def create_wind_output(cutout: geodata.Cutout, turbine, smooth, var_height): - ds_wind = cutout.wind(turbine, smooth, var_height=var_height) - df_wind = ds_wind.to_dataframe(name="wind") - return df_wind - - -def create_windspd_output(cutout: geodata.Cutout, turbine, var_height): - ds_windspd = cutout.windspd(turbine=turbine, var_height=var_height) - df_windspd = ds_windspd.to_dataframe(name="windspd") - return df_windspd - - -def create_windwpd_output(cutout: geodata.Cutout, turbine, var_height): - ds_windwpd = cutout.windwpd(turbine=turbine, var_height=var_height) - df_windwpd = ds_windwpd.to_dataframe(name="windwpd") - return df_windwpd - - def test_download(): configs = get_data_configs() years = get_years() @@ -117,89 +59,4 @@ def test_download(): for config, year, month, bound in zip(configs, years, months, bounds): dataset = get_merra2(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_merra2(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] - - -def test_cutout(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - assert cutout.prepared - - -def test_wind_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - smooth = get_smooth() - var_height = get_var_height() - df_wind = create_wind_output(cutout, turbine, smooth, var_height) - assert df_wind.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "wind": np.dtype("float64"), - } - - -def test_windspd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - var_height = get_var_height() - df_windspd = create_windspd_output(cutout, turbine, var_height) - assert df_windspd.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "windspd": np.dtype("float32"), - } - - -def test_windwpd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - var_height = get_var_height() - df_windwpd = create_windwpd_output(cutout, turbine, var_height) - assert df_windwpd.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "windwpd": np.dtype("float32"), - } + assert dataset.downloaded From de437697b4d69d6457524dd848f443829d3a62c6 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 9 Jun 2025 16:26:58 -0700 Subject: [PATCH 51/54] fix: typo --- src/geodata/datasets/merra2/_base.py | 1 - src/geodata/model/results/_base.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index bd99e79b..d5fc7639 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -253,7 +253,6 @@ def tasks_func( case "monthly": return [ dict( - prepare_func=prepare_func, xs=xs, ys=ys, year=year, diff --git a/src/geodata/model/results/_base.py b/src/geodata/model/results/_base.py index 95cb0f31..2430f01d 100644 --- a/src/geodata/model/results/_base.py +++ b/src/geodata/model/results/_base.py @@ -21,13 +21,15 @@ import logging from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING import xarray as xr from geodata.config import model_dir if TYPE_CHECKING: + from typing import Self + from .._base import BaseModel logger = logging.getLogger(__name__) From 6dc98028ee4873e8093edc0c0e2a73bdf8185ed6 Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Mon, 9 Jun 2025 21:17:57 -0700 Subject: [PATCH 52/54] fix: CI update --- .github/workflows/dev_test.yml | 2 +- .github/workflows/pr_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_test.yml b/.github/workflows/dev_test.yml index cdbcf0cf..8e78dc0c 100644 --- a/.github/workflows/dev_test.yml +++ b/.github/workflows/dev_test.yml @@ -22,7 +22,7 @@ jobs: max-parallel: 10 fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.10.11, 3.12.6] # These versions are compatibale with all three OS on GitHub Actions steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 9637c60d..4ec57294 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -22,7 +22,7 @@ jobs: max-parallel: 10 fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.10.11, 3.12.6] # These versions are compatibale with all three OS on GitHub Actions steps: - uses: actions/checkout@v3 From edf86019a01518b889b52a3f7a025fbb6a1c1d7a Mon Sep 17 00:00:00 2001 From: Xiqiang Liu <9440183+xiqiangliu@users.noreply.github.com> Date: Wed, 11 Jun 2025 22:13:12 -0700 Subject: [PATCH 53/54] docs: add more extensive docs on wind estimation --- docs/source/index.rst | 18 ++-- docs/source/modeling/wind/extrapolation.rst | 75 ++++++++++---- docs/source/modeling/wind/index.rst | 55 +++++++++- docs/source/modeling/wind/interpolation.rst | 109 ++++++++++++++++---- src/geodata/resource.py | 13 +++ 5 files changed, 219 insertions(+), 51 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 4674cc80..fc398a3b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -26,6 +26,15 @@ Welcome to Geodata's documentation! datasets/merra2/index datasets/* +.. toctree:: + :maxdepth: 1 + :caption: Modeling + :glob: + :hidden: + + modeling/wind/index + modeling/* + .. toctree:: :maxdepth: 1 :caption: Mask @@ -57,15 +66,6 @@ Welcome to Geodata's documentation! .. application/* -.. toctree:: - :maxdepth: 1 - :caption: Modeling - :glob: - :hidden: - - modeling/wind/index - modeling/* - .. toctree:: :maxdepth: 1 :caption: API Reference diff --git a/docs/source/modeling/wind/extrapolation.rst b/docs/source/modeling/wind/extrapolation.rst index af92a2b4..a983806e 100644 --- a/docs/source/modeling/wind/extrapolation.rst +++ b/docs/source/modeling/wind/extrapolation.rst @@ -1,7 +1,8 @@ Tutorial: Estimate Wind Speed with Extrapolation ================================================ -In this tutorial, we will learn how to estimate wind speed using the extrapolation model from the geodata library. +In this tutorial, we will learn how to estimate wind speed using the extrapolation model + from the geodata library. .. warning:: Performing wind speed estimation using extrapolation requires a dataset with known @@ -20,11 +21,12 @@ To get started, we need to import the required libraries. We will import the `Wi .. code:: Python - import geodata import xarray as xr + from geodata.datasets import load_dataset from geodata.model.wind import WindExtrapolationModel + Step 2: Load the dataset ------------------------ @@ -33,20 +35,25 @@ Next, we need to load the dataset that contains the wind speed data. We will use .. code:: Python # Load the dataset - ds = geodata.Dataset( - module="merra2", - weather_data_config="slv_flux_hourly", - years=slice(2010, 2010), + ds_cls = load_dataset("slv_flux_hourly") + ds = ds_cls( + years=slice(2006, 2006), months=slice(1, 1), - bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box ) - if not ds.prepared: - ds.get_data() # Download the data if we don't have it locally + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + Step 3: Compute extrapolation parameters -------------------------------------------- -The extrapolation is separated into two steps, first estimating extrapolation parameters using linear regression, and second extrapolating to desired heights). First, we compute the extrapolation parameters. For more information on the model, see below "How the Extrapolation Model Works". +The extrapolation is separated into two steps, first estimating extrapolation parameters +using linear regression, and second extrapolating to desired heights. +First, we compute the extrapolation parameters. +For more information on the model, see the section below: `How the Extrapolation Model Works`_. .. code:: Python @@ -57,7 +64,8 @@ The extrapolation is separated into two steps, first estimating extrapolation pa model = WindExtrapolationModel(ds) model.prepare() -If you have already prepared a cutout with the config :code:`slv_flux_hourly`, you can also pass +If you have already prepared a cutout with the config :code:`slv_flux_hourly`, you +can also pass that into the model as well. The model treats dataset and cutouts indifferently. Simply replace :code:`ds` with your cutout variable. @@ -72,11 +80,11 @@ Simply replace :code:`ds` with your cutout variable. point on, you can load and use the model directly without re-preparing it. Step 4: Estimate using the extrapolation model ----------------------------------- +---------------------------------------------- Now that we have prepared the model, we can perform the extrapolation to estimate wind speed at the desired locations. Suppose we want to estimate the wind speed at a height -of 60 above ground during Jaunary of 2006 for the entire region covered by the original +of 60 above ground during January of 2006 for the entire region covered by the original dataset, we can do this as follows: .. code:: Python @@ -91,13 +99,40 @@ This will return an xarray DataArray containing the estimated wind speed values. that you can also select a subset area by passing in :code:`xs=slice(start, end)` and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. -.. note:: - As the underlying MERRA2 dataset already contained wind speed at certain heights, the - model also has a feature to return the original wind speed values from the dataset - if desired. To do this, simply set the `use_real_data` parameter to `True` in the - `estimate` method. You do not need worry about whether the height you queried is - available in the dataset; the model will handle that for you. If the height is not - available, it will perform extrapolation instead. + +Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model +-------------------------------------------------------------------------------- + +Geodata also supports a limited set of wind turbine models to estimate the capacity +factor (CF) of a wind turbine directly. To get a list of available wind turbine models, +you can use the `get_available_windturbines` function: + +.. code:: Python + + from geodata.resource import get_available_windturbines + + turbines = get_available_windturbines() + print(turbines) # List of available wind turbine configurations + + +To estimate the capacity factor of a wind turbine, you can use the `estimate` method +and passign in the `turbine` parameter with the name of the wind turbine model. + +.. code:: Python + + # Estimate the capacity factor for a specific wind turbine model + estimated_cf: xr.Dataset = model.estimate( + turbine="Vestas_V112_3MW", # Example wind turbine model + years=slice(2006, 2006), + months=slice(1, 1), + ) + + print(estimated_cf) # Display the estimated capacity factor + + +The output will be an xarray Dataset containing the estimated capacity factor values +for the specified wind turbine model over the given time period and region. + How the Extrapolation Model Works --------------------------------- diff --git a/docs/source/modeling/wind/index.rst b/docs/source/modeling/wind/index.rst index 234ba7ca..e6a93019 100644 --- a/docs/source/modeling/wind/index.rst +++ b/docs/source/modeling/wind/index.rst @@ -8,8 +8,9 @@ estimate wind speed with two modes: interpolation and extrapolation. How to use the models --------------------- -Unlike other modules of geodata, the model module does not get imported automatically. -In other words, one cannot use the models with an import statement like this: +Unlike some other modules of geodata, the model module does not get imported +automatically. In other words, one cannot use the models with an import statement +like this: .. code:: Python @@ -26,8 +27,54 @@ Instead, the user must import the models explicitly: The reason for this is to keep the main geodata namespace clean, since we might add many more models in the future. -More details on how to use the models can be found in each model's respective tutorial -as well as in the API reference. +In general, models are created based off of a dataset object, we must be downloaded +first. We'll use the `wind_3d_hourly` dataset from the ERA5 dataset as an example: + +.. code:: Python + + from geodata.datasets import load_dataset + from geodata.model.wind import WindInterpolationModel + + # Load the dataset + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( + years=slice(2006, 2006), + months=slice(1, 1), + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + ) + + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + + +Once we have the dataset, we can create a model based on it. +The model will be associated with the dataset forever, so if you wish to use a +different dataset, you will need to create a new model. + +.. code:: Python + + # Create a model based on the above dataset + model = WindInterpolationModel(ds) + model.prepare() # Prepare the model + + print(model.prepared) # Check if the model is prepared. Should return True. + + +Once the model is prepared, we can use it to estimate wind speed at desired heights. + +.. code:: Python + + # Estimate wind speed at a specific height (e.g., 100 meters) + wind_speed: xr.Dataset = model.estimate(height=100.0) + + # The wind_speed variable is an xarray Dataset containing the estimated wind speed at the specified height. + # Over the region covered by the original dataset. + + +The above demonstrates the typical workflow. More model-specific details can be found +in each model's respective tutorial as well as in the API reference. .. toctree:: :maxdepth: 1 diff --git a/docs/source/modeling/wind/interpolation.rst b/docs/source/modeling/wind/interpolation.rst index d946a1f4..788bba56 100644 --- a/docs/source/modeling/wind/interpolation.rst +++ b/docs/source/modeling/wind/interpolation.rst @@ -1,7 +1,8 @@ Tutorial: Estimate Wind Speed with Interpolation ================================================ -In this tutorial, we will learn how to estimate wind speed using the interpolation model from the geodata library. +In this tutorial, we will learn how to estimate wind speed using the interpolation model + from the geodata library. .. warning:: Performing wind speed estimation using interpolation requires a dataset with known @@ -16,37 +17,75 @@ In this tutorial, we will learn how to estimate wind speed using the interpolati Step 1: Import the necessary libraries ---------------------------------------- -To get started, we need to import the required libraries. We will import the `WindInterpolationModel` from the `geodata` library, as well as any other libraries needed for data handling and visualization. +To get started, we need to import the required libraries. We will import +the `WindInterpolationModel` from the `geodata` library, as well as any other +libraries needed for data handling and visualization. + .. code:: Python - import geodata import xarray as xr + from geodata.datasets import load_dataset from geodata.model.wind import WindInterpolationModel + Step 2: Load the dataset ------------------------ -Next, we need to load the dataset that contains the wind speed data. We will use the `wind_3d_hourly` dataset from the ERA5 dataset. +Next, we need to load the dataset that contains the wind speed data. +We will use the `wind_3d_hourly` dataset from the ERA5 dataset. + .. code:: Python # Load the dataset - ds = geodata.Dataset( - module="era5", - weather_data_config="wind_3d_hourly", + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( years=slice(2006, 2006), months=slice(1, 1), - bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box ) - if not ds.prepared: - ds.get_data() # Download the data if we don't have it locally + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + Step 3: Compute interpolation parameters -------------------------------------------- -The interpolation is separated into two steps to separate the computationally-intensive step (estimating interpolation parameters) from the computationally-easy step (interpolating at desired heights). First, we compute the interpolation parameters. +The interpolation is separated into two steps to separate the computationally-intensive +step (estimating interpolation parameters) from the computationally-easy step +(interpolating at desired heights). + +.. note:: + Wind-speed estimation using **cubic spline interpolation** fits a smooth, piecewise cubic polynomial + to known wind-speed data across spatial or temporal dimensions. Given a set of data points + :math:`(x_0, y_0), (x_1, y_1), \dots, (x_n, y_n)`, where :math:`x_i` are known positions (e.g., time or altitude) + and :math:`y_i` are corresponding wind speeds, the cubic spline for each interval + :math:`[x_i, x_{i+1}]` is defined as: + + .. math:: + + S_i(x) = a_i + b_i(x - x_i) + c_i(x - x_i)^2 + d_i(x - x_i)^3 + + The coefficients :math:`a_i, b_i, c_i, d_i` are determined by solving a system of equations subject to: + + 1. **Interpolation condition**: + :math:`S_i(x_i) = y_i`, and :math:`S_i(x_{i+1}) = y_{i+1}` + 2. **Continuity of first derivative**: + :math:`S_i'(x_{i+1}) = S_{i+1}'(x_{i+1})` + 3. **Continuity of second derivative**: + :math:`S_i''(x_{i+1}) = S_{i+1}''(x_{i+1})` + 4. **Boundary conditions**, typically *natural*: + :math:`S_0''(x_0) = 0`, and :math:`S_{n-1}''(x_n) = 0` + + The resulting spline provides a smooth and continuous estimate of wind speed, allowing accurate + interpolation between measured data points. With `wind_3d_hourly`, we have wind speeds at seven different heights AGL. + This enables us to estimate wind speeds at any height AGL within that range (from approximately 10m AGL to 170m AGL). + +First, we compute the interpolation parameters. .. code:: Python @@ -71,22 +110,56 @@ Simply replace :code:`ds` with your cutout variable. interpolation. You only need to call it once after loading the dataset. From that point on, you can load and use the model directly without re-preparing it. -Step 4: Estimate using the interpolation model ----------------------------------- +Step 4: Estimate wind speeds using the interpolation model +---------------------------------------------------------- Now that we have prepared the model, we can perform the interpolation to estimate wind speed at the desired locations. Suppose we want to estimate the wind speed at a height -of 60 m above ground during Jaunary of 2006 for the entire region covered by the original -dataset, we can do this as follows: +of 60 m above ground during January of 2006 for the entire region covered +by the original dataset, we can do this as follows: .. code:: Python - estimated_wind_speed = model.estimate( - height=60, + estimated_wind_speed: xr.Dataset = model.estimate( + height=60.0, years=slice(2006, 2006), months=slice(1, 1), ) -This will return an xarray DataArray containing the estimated wind speed values. Note + +This will return an xarray Dataset containing the estimated wind speed values. Note that you can also select a subset area by passing in :code:`xs=slice(start, end)` and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. + +Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model +-------------------------------------------------------------------------------- + +Geodata also supports a limited set of wind turbine models to estimate the capacity +factor (CF) of a wind turbine directly. To get a list of available wind turbine models, +you can use the `get_available_windturbines` function: + +.. code:: Python + + from geodata.resource import get_available_windturbines + + turbines = get_available_windturbines() + print(turbines) # List of available wind turbine configurations + + +To estimate the capacity factor of a wind turbine, you can use the `estimate` method +and passign in the `turbine` parameter with the name of the wind turbine model. + +.. code:: Python + + # Estimate the capacity factor for a specific wind turbine model + estimated_cf: xr.Dataset = model.estimate( + turbine="Vestas_V112_3MW", # Example wind turbine model + years=slice(2006, 2006), + months=slice(1, 1), + ) + + print(estimated_cf) # Display the estimated capacity factor + + +The output will be an xarray Dataset containing the estimated capacity factor values +for the specified wind turbine model over the given time period and region. diff --git a/src/geodata/resource.py b/src/geodata/resource.py index 28a9847d..86a12ad3 100644 --- a/src/geodata/resource.py +++ b/src/geodata/resource.py @@ -23,6 +23,7 @@ import logging from operator import itemgetter + import numpy as np import yaml from scipy.signal import fftconvolve @@ -160,3 +161,15 @@ def smooth(velocities, power): ) return turbine + + +def get_available_windturbines(): + """Get a list of available wind turbine configurations.""" + res_path = SRC_ROOT / "resources/windturbine" + return [p.stem for p in res_path.glob("*.yaml") if p.is_file()] + + +def get_available_solarpanels(): + """Get a list of available solar panel configurations.""" + res_path = SRC_ROOT / "resources/solarpanel" + return [p.stem for p in res_path.glob("*.yaml") if p.is_file()] From 2db43e32a0f17d14182e5c99d2b949ee66c0a755 Mon Sep 17 00:00:00 2001 From: Michael Janelle Date: Mon, 25 Aug 2025 16:23:14 -0700 Subject: [PATCH 54/54] ERA5: fix 2m dewpoint key (2m_dewpoint_temperature) in wind_solar.py Fixed typo with variable "d2m". Now reads 2m_dewpoint_temperature instead of 2m_dew_point_temperature --- src/geodata/datasets/era5/hourly/wind_solar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py index 9ca2da1c..8b139951 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -43,7 +43,7 @@ class ERA5WindSolarHourlyDataset(ERA5BaseDataset): - 100m_u_component_of_wind - 100m_v_component_of_wind - 2m_temperature - - 2m_dew_point_temperature + - 2m_dewpoint_temperature - runoff - soil_temperature_level_4 - surface_net_solar_radiation @@ -57,12 +57,12 @@ class ERA5WindSolarHourlyDataset(ERA5BaseDataset): weather_config = "wind_solar_hourly" - # Information that are needed for ERA5's API request + # Information that is needed for ERA5's API request variables = { "100m_u_component_of_wind": "u100", "100m_v_component_of_wind": "v100", "2m_temperature": "t2m", - "2m_dew_point_temperature": "d2m", + "2m_dewpoint_temperature": "d2m", "runoff": "ro", "soil_temperature_level_4": "stl4", "surface_net_solar_radiation": "ssr",