diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..5266e394 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/devcontainers/base:bookworm + +USER root +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends libgdal-dev \ + && rm -rf /var/lib/apt/lists/* + +USER vscode +RUN curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..76ba6bc0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/debian +{ + "name": "Debian", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "build": { + // Path is relative to the devcontainer.json file. + "dockerfile": "Dockerfile" + }, + "postCreateCommand": "uv sync", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "charliermarsh.ruff" + ] + } + }, + "mounts": [ + "source=${localEnv:HOME}${localEnv:USERPROFILE}/.local/geodata,target=/home/vscode/.local/geodata,type=bind" + ] + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Configure tool-specific properties. + // "customizations": {}, + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.github/workflows/dev_test.yml b/.github/workflows/dev_test.yml index cdbcf0cf..8e78dc0c 100644 --- a/.github/workflows/dev_test.yml +++ b/.github/workflows/dev_test.yml @@ -22,7 +22,7 @@ jobs: max-parallel: 10 fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.10.11, 3.12.6] # These versions are compatibale with all three OS on GitHub Actions steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 9637c60d..4ec57294 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -22,7 +22,7 @@ jobs: max-parallel: 10 fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.10.11, 3.12.6] # These versions are compatibale with all three OS on GitHub Actions steps: - uses: actions/checkout@v3 diff --git a/docs/source/application/wind_estimation.ipynb b/docs/source/application/wind_estimation.ipynb deleted file mode 100644 index 82536b92..00000000 --- a/docs/source/application/wind_estimation.ipynb +++ /dev/null @@ -1,657 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Example: Use Geodata to Estimate Wind Speed and Direction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Geodata can be used for many applications. In this example, we will use the Geodata to estimate the\n", - "wind speed and direction at a given location." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import Necessary Packages" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "import geodata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def log_ratio(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic ratio law\n", - " Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " \"\"\"\n", - "\n", - " wnd_spd = ds[from_name] * (\n", - " np.log(to_height / ds[\"roughness\"]) / np.log(ds[from_height] / ds[\"roughness\"])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log ratio\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " \"\"\"\n", - " vonk = 0.4\n", - " wnd_spd = ds[from_name] + (ds[\"ustar\"] / vonk * np.log((to_height - ds[\"disph\"]) / ds[from_height]))\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "## Flux stability correction functions\n", - "def psi_linear(z, ds):\n", - " \"\"\"NOTE: This function does not perform well for high z/L\n", - " Linear stability correction [1,2]\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " Eval = \t0, if z/L <=0\n", - " linear if z/L > 0\n", - " [1] Businger, J.A., Wyngaard, J.C., Izumi, Y., Bradley, E.F., 1971. Flux profile relationships in the atmospheric surface layer. J. Atmos. Sci. 28, 181–189.\n", - " [2] Dyer, A.J., 1974. A review of flux–profile relationships. Boundary-Layer Meteorol. 7, 363–372.\n", - " \"\"\"\n", - " beta = 5.2\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[0 < ds[\"a\"]] = -beta * ds[\"a\"].values[0 < ds[\"a\"]]\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexp(z, ds):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2])\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexpconst(z, ds, const=7):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2]) with constant plateau\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - " const = upper bound of z/L, after which = constant\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] > const] = -A * (const + B * (const - C / D) * np.exp(-D * const) + B * C / D)\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def L_vph(ds):\n", - " \"\"\"Obuhkov length using virtual potential heat flux term [1] (described in detail in SI [2])\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " grav = 9.81 # gravitational acceleration in kg m s-2\n", - " CPD = 1004 # specific heat of dry air at constant pressure J K-1 kg-1\n", - " Le = 2.257e6 # latent heat of evaporation [J/kg]\n", - " kp = 2 / 7 # Poisson constant\n", - " Rd = 287 # Ideal gas constant [J/kg/K]\n", - " p0 = 1e5 # standard air pressure\n", - "\n", - " ds[\"p\"] = ds[\"rhoa\"] * Rd * ds[\"tlml\"]\n", - " ds[\"vphflux\"] = ds[\"hflux\"] + 0.61 * CPD / Le * ds[\"tlml\"] * (p0 / ds[\"p\"]) ** kp * ds[\"eflux\"]\n", - " ds[\"L\"] = -(ds[\"tlml\"] * ds[\"ustar\"] ** 3 * CPD * ds[\"rhoa\"]) / (vonk * grav * ds[\"vphflux\"])\n", - " return ds[\"L\"]\n", - "\n", - "\n", - "def winddir(ds):\n", - " \"\"\"Wind direction using lowest model layer\"\"\"\n", - "\n", - " ds[\"winddir\"] = np.degrees(np.arctan(ds[\"ulml\"] / ds[\"vlml\"]))\n", - " ds[\"winddir\"].values[ds[\"vlml\"] < 0] += 180\n", - " ds[\"winddir\"].values[(ds[\"vlml\"] > 0) & (ds[\"ulml\"] < 0)] += 360\n", - " return ds[\"winddir\"]\n", - "\n", - "\n", - "def _log_law_flux(ds, to_height, from_height, from_name, psifn, Lfn=L_vph): # pylint: disable=unused-argument\n", - " \"\"\"Compute logarithmic (integration) law given stability correction fn in terms of Obukhov length (derived from heat flux) [1]\n", - " Called by: log_law_flux_**\n", - "\n", - " [1] Sharan, M., & Aditi. (2009).\n", - " Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions.\n", - " Atmospheric Research, 94(2), 246–253.\n", - " https://doi.org/10.1016/j.atmosres.2009.05.014\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " ds[\"L\"] = L_vph(ds)\n", - "\n", - " wnd_spd = ds[from_name] + ds[\"ustar\"] / vonk * (\n", - " np.log((to_height - ds[\"disph\"]) / ds[from_height]) - psifn(to_height, ds[[\"L\", \"roughness\"]])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law and stability correction {psifn}\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law_flux_linear(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with linear stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linear)\n", - "\n", - "\n", - "def log_law_flux_linearexp(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexp)\n", - "\n", - "\n", - "def log_law_flux_linearexpconst(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential-constant stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexpconst)\n", - "\n", - "\n", - "\"\"\"\n", - "Main call (from convert.convert_wind)\n", - "\"\"\"\n", - "\n", - "\n", - "def extrapolate_wind_speed(ds, to_height, extrap_fn=log_ratio, from_height=None, var_height=None):\n", - " \"\"\"Extrapolate the wind speed from a given height above ground to another.\n", - "\n", - " If ds already contains a key refering to wind speeds at the desired to_height,\n", - " no conversion is done and the wind speeds are directly returned.\n", - "\n", - " Otherwise, extrapolates according to (1) extrap_fn and (2) heights\n", - "\n", - "\n", - "\n", - " Parameters\n", - " ----------\n", - " ds : xarray.Dataset\n", - " Dataset containing the wind speed time-series\n", - " to_height : int|float\n", - " Height (m) to which the wind speeds are extrapolated\n", - " extrap_fn : function for wind speed extrapolation\n", - " log_ratio : wind speed follows the logarithmic ratio law as desribed in [1]\n", - " log_law : wind speed follows logarithmic (integration) law described in [3]\n", - " power_law : wind speed follows power law (with fixed alpha), e.g., in [4]\n", - " from_height : int\n", - " (Optional)\n", - " Height (m) from which the wind speeds are interpolated to 'to_height'.\n", - " If not provided, the closest height to 'to_height' is selected.\n", - " var_height : str\n", - " (Optional)\n", - " suffix of variables in ds corresponding to variable height\n", - " e.g., `lml` => height contained in `hlml`, wind speed contained in `wndlml`\n", - "\n", - " Returns\n", - " -------\n", - " da : xarray.DataArray\n", - " DataArray containing the extrapolated wind speeds. Name of the DataArray\n", - " is 'wnd{to_height:d}'.\n", - "\n", - " References\n", - " ----------\n", - " [1] Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " [2] https://en.wikipedia.org/w/index.php?title=Roughness_length&oldid=862127433,\n", - " Retrieved 2019-02-15.\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " [4] Archer, C.L., Jacobson, M.Z., 2005. Evaluation of global wind power.\n", - " Journal of Geophysical Research 110, D12110.\n", - " \"\"\"\n", - "\n", - " to_name = \"wnd{h:0d}m\".format(h=int(to_height))\n", - " if to_name in ds:\n", - " # already found wind speed at given height in dataset\n", - " return ds[to_name]\n", - "\n", - " # Sanitize roughness for logarithm: 0.0002 corresponds to open water [2]\n", - " ds[\"roughness\"].values[ds[\"roughness\"].values <= 0.0] = 0.0002\n", - "\n", - " if not from_height is None:\n", - " # passed a from_height\n", - " if not var_height is None:\n", - " raise AssertionError(\"Cannot pass both from_height and var_height to extrapolate_wind_speed\")\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = wnd_spd.attrs[\"long_name\"] + \", \" + f\"from fixed height = {from_height}\"\n", - "\n", - " elif not var_height is None:\n", - " # passed a variable height (eg lml)\n", - " # set variable names\n", - " from_height = f\"h{var_height}\"\n", - " from_name = f\"wnd{var_height}\"\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, from_height, from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from variable height = {var_height}\"\n", - " )\n", - "\n", - " else:\n", - " # based on nearest height\n", - " heights = np.asarray([int(s[3:-1]) for s in ds if s.startswith(\"wnd\")])\n", - " if len(heights) == 0:\n", - " raise AssertionError(\"Wind speed is not in dataset\")\n", - "\n", - " from_height = heights[np.argmin(np.abs(heights - to_height))]\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from nearest height = {from_height}\"\n", - " )\n", - "\n", - " return wnd_spd.rename(to_name)\n", - "## Copyright 2020 Michael Davidson (UCSD).\n", - "\n", - "## This program is free software; you can redistribute it and/or\n", - "## modify it under the terms of the GNU General Public License as\n", - "## published by the Free Software Foundation; either version 3 of the\n", - "## License, or (at your option) any later version.\n", - "\n", - "## This program is distributed in the hope that it will be useful,\n", - "## but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "## GNU General Public License for more details.\n", - "\n", - "## You should have received a copy of the GNU General Public License\n", - "## along with this program. If not, see .\n", - "\n", - "\n", - "\"\"\"\n", - "GEODATA\n", - "\n", - "Geospatial Data Collection and \"Pre-Analysis\" Tools\n", - "\"\"\"\n", - "import logging\n", - "\n", - "import numpy as np\n", - "\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "\n", - "\"\"\"\n", - "Extrapolation functions\n", - "\tNote: these are called internally\n", - "\"\"\"\n", - "\n", - "\n", - "def log_ratio(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic ratio law\n", - " # Equation (2) in Andresen, G. et al (2015):\n", - " # \t'Validation of Danish wind time series from a new global renewable\n", - " # \tenergy atlas for energy system analysis'.\n", - " \"\"\"\n", - "\n", - " wnd_spd = ds[from_name] * (\n", - " np.log(to_height / ds[\"roughness\"]) / np.log(ds[from_height] / ds[\"roughness\"])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log ratio\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " \"\"\"\n", - " vonk = 0.4\n", - " wnd_spd = ds[from_name] + (ds[\"ustar\"] / vonk * np.log((to_height - ds[\"disph\"]) / ds[from_height]))\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "## Flux stability correction functions\n", - "def psi_linear(z, ds):\n", - " \"\"\"NOTE: This function does not perform well for high z/L\n", - " Linear stability correction [1,2]\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " Eval = \t0, if z/L <=0\n", - " linear if z/L > 0\n", - " [1] Businger, J.A., Wyngaard, J.C., Izumi, Y., Bradley, E.F., 1971. Flux profile relationships in the atmospheric surface layer. J. Atmos. Sci. 28, 181–189.\n", - " [2] Dyer, A.J., 1974. A review of flux–profile relationships. Boundary-Layer Meteorol. 7, 363–372.\n", - " \"\"\"\n", - " beta = 5.2\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[0 < ds[\"a\"]] = -beta * ds[\"a\"].values[0 < ds[\"a\"]]\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexp(z, ds):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2])\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def psi_linearexpconst(z, ds, const=7):\n", - " \"\"\"Linear-exponential piecewise stability correction [1] (repeated in [2]) with constant plateau\n", - " z = to_height\n", - " ds = dataset with variables: L\n", - " const = upper bound of z/L, after which = constant\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " aconst = 5\n", - " A = 1\n", - " B = 2 / 3\n", - " C = 5\n", - " D = 0.35\n", - " ds[\"a\"] = z / ds[\"L\"]\n", - " ds[\"psim\"] = ds[\"a\"] * 0 # create zeroes same length as dataset\n", - " ds[\"psim\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)] = (\n", - " -aconst * ds[\"a\"].values[(0 < ds[\"a\"]) & (ds[\"a\"] <= 0.5)]\n", - " )\n", - " ds[\"psim\"].values[0.5 < ds[\"a\"]] = -A * (\n", - " ds[\"a\"].values[0.5 < ds[\"a\"]]\n", - " + B * (ds[\"a\"].values[0.5 < ds[\"a\"]] - C / D) * np.exp(-D * ds[\"a\"].values[0.5 < ds[\"a\"]])\n", - " + B * C / D\n", - " )\n", - " ds[\"psim\"].values[ds[\"a\"] > const] = -A * (const + B * (const - C / D) * np.exp(-D * const) + B * C / D)\n", - " ds[\"psim\"].values[ds[\"a\"] <= 0] = 0\n", - " return ds[\"psim\"]\n", - "\n", - "\n", - "def L_vph(ds):\n", - " \"\"\"Obuhkov length using virtual potential heat flux term [1] (described in detail in SI [2])\n", - "\n", - " [1] Emeis, S. (2013). Wind Energy Meteorology. Retrieved from http://link.springer.com/10.1007/978-3-642-30523-8 (Note: error in eq 3.21)\n", - " [2] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165.\n", - " https://doi.org/10.1016/j.renene.2016.03.028\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " grav = 9.81 # gravitational acceleration in kg m s-2\n", - " CPD = 1004 # specific heat of dry air at constant pressure J K-1 kg-1\n", - " Le = 2.257e6 # latent heat of evaporation [J/kg]\n", - " kp = 2 / 7 # Poisson constant\n", - " Rd = 287 # Ideal gas constant [J/kg/K]\n", - " p0 = 1e5 # standard air pressure\n", - "\n", - " ds[\"p\"] = ds[\"rhoa\"] * Rd * ds[\"tlml\"]\n", - " ds[\"vphflux\"] = ds[\"hflux\"] + 0.61 * CPD / Le * ds[\"tlml\"] * (p0 / ds[\"p\"]) ** kp * ds[\"eflux\"]\n", - " ds[\"L\"] = -(ds[\"tlml\"] * ds[\"ustar\"] ** 3 * CPD * ds[\"rhoa\"]) / (vonk * grav * ds[\"vphflux\"])\n", - " return ds[\"L\"]\n", - "\n", - "\n", - "def winddir(ds):\n", - " \"\"\"Wind direction using lowest model layer\"\"\"\n", - "\n", - " ds[\"winddir\"] = np.degrees(np.arctan(ds[\"ulml\"] / ds[\"vlml\"]))\n", - " ds[\"winddir\"].values[ds[\"vlml\"] < 0] += 180\n", - " ds[\"winddir\"].values[(ds[\"vlml\"] > 0) & (ds[\"ulml\"] < 0)] += 360\n", - " return ds[\"winddir\"]\n", - "\n", - "\n", - "def _log_law_flux(ds, to_height, from_height, from_name, psifn, Lfn=L_vph): # pylint: disable=unused-argument\n", - " \"\"\"Compute logarithmic (integration) law given stability correction fn in terms of Obukhov length (derived from heat flux) [1]\n", - " Called by: log_law_flux_**\n", - "\n", - " [1] Sharan, M., & Aditi. (2009).\n", - " Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions.\n", - " Atmospheric Research, 94(2), 246–253.\n", - " https://doi.org/10.1016/j.atmosres.2009.05.014\n", - " \"\"\"\n", - " vonk = 0.4 # Von Karman constant\n", - " ds[\"L\"] = L_vph(ds)\n", - "\n", - " wnd_spd = ds[from_name] + ds[\"ustar\"] / vonk * (\n", - " np.log((to_height - ds[\"disph\"]) / ds[from_height]) - psifn(to_height, ds[[\"L\", \"roughness\"]])\n", - " )\n", - " wnd_spd.attrs.update(\n", - " {\n", - " \"long_name\": f\"extrapolated {to_height} m wind speed using log (integration) law and stability correction {psifn}\",\n", - " \"units\": \"m s**-1\",\n", - " }\n", - " )\n", - " return wnd_spd\n", - "\n", - "\n", - "def log_law_flux_linear(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with linear stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linear)\n", - "\n", - "\n", - "def log_law_flux_linearexp(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexp)\n", - "\n", - "\n", - "def log_law_flux_linearexpconst(ds, to_height, from_height, from_name):\n", - " \"\"\"Logarithmic (integration) law with piecewise linear-exponential-constant stability correction in terms of Obukhov length\"\"\"\n", - " return _log_law_flux(ds, to_height, from_height, from_name, psi_linearexpconst)\n", - "\n", - "\n", - "\"\"\"\n", - "Main call (from convert.convert_wind)\n", - "\"\"\"\n", - "\n", - "\n", - "def extrapolate_wind_speed(ds, to_height, extrap_fn=log_ratio, from_height=None, var_height=None):\n", - " \"\"\"Extrapolate the wind speed from a given height above ground to another.\n", - "\n", - " If ds already contains a key refering to wind speeds at the desired to_height,\n", - " no conversion is done and the wind speeds are directly returned.\n", - "\n", - " Otherwise, extrapolates according to (1) extrap_fn and (2) heights\n", - "\n", - "\n", - "\n", - " Parameters\n", - " ----------\n", - " ds : xarray.Dataset\n", - " Dataset containing the wind speed time-series\n", - " to_height : int|float\n", - " Height (m) to which the wind speeds are extrapolated\n", - " extrap_fn : function for wind speed extrapolation\n", - " log_ratio : wind speed follows the logarithmic ratio law as desribed in [1]\n", - " log_law : wind speed follows logarithmic (integration) law described in [3]\n", - " power_law : wind speed follows power law (with fixed alpha), e.g., in [4]\n", - " from_height : int\n", - " (Optional)\n", - " Height (m) from which the wind speeds are interpolated to 'to_height'.\n", - " If not provided, the closest height to 'to_height' is selected.\n", - " var_height : str\n", - " (Optional)\n", - " suffix of variables in ds corresponding to variable height\n", - " e.g., `lml` => height contained in `hlml`, wind speed contained in `wndlml`\n", - "\n", - " Returns\n", - " -------\n", - " da : xarray.DataArray\n", - " DataArray containing the extrapolated wind speeds. Name of the DataArray\n", - " is 'wnd{to_height:d}'.\n", - "\n", - " References\n", - " ----------\n", - " [1] Equation (2) in Andresen, G. et al (2015):\n", - " 'Validation of Danish wind time series from a new global renewable\n", - " energy atlas for energy system analysis'.\n", - " [2] https://en.wikipedia.org/w/index.php?title=Roughness_length&oldid=862127433,\n", - " Retrieved 2019-02-15.\n", - " [3] S. Emeis, Wind Energy Meteorology (Springer, Berlin, 2013).\n", - " [4] Archer, C.L., Jacobson, M.Z., 2005. Evaluation of global wind power.\n", - " Journal of Geophysical Research 110, D12110.\n", - " \"\"\"\n", - "\n", - " to_name = \"wnd{h:0d}m\".format(h=int(to_height))\n", - " if to_name in ds:\n", - " # already found wind speed at given height in dataset\n", - " return ds[to_name]\n", - "\n", - " # Sanitize roughness for logarithm: 0.0002 corresponds to open water [2]\n", - " ds[\"roughness\"].values[ds[\"roughness\"].values <= 0.0] = 0.0002\n", - "\n", - " if not from_height is None:\n", - " # passed a from_height\n", - " if not var_height is None:\n", - " raise AssertionError(\"Cannot pass both from_height and var_height to extrapolate_wind_speed\")\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = wnd_spd.attrs[\"long_name\"] + \", \" + f\"from fixed height = {from_height}\"\n", - "\n", - " elif not var_height is None:\n", - " # passed a variable height (eg lml)\n", - " # set variable names\n", - " from_height = f\"h{var_height}\"\n", - " from_name = f\"wnd{var_height}\"\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, from_height, from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from variable height = {var_height}\"\n", - " )\n", - "\n", - " else:\n", - " # based on nearest height\n", - " heights = np.asarray([int(s[3:-1]) for s in ds if s.startswith(\"wnd\")])\n", - " if len(heights) == 0:\n", - " raise AssertionError(\"Wind speed is not in dataset\")\n", - "\n", - " from_height = heights[np.argmin(np.abs(heights - to_height))]\n", - " from_name = \"wnd{h:0d}m\".format(h=int(from_height))\n", - " ds[\"from_height\"] = from_height\n", - "\n", - " wnd_spd = extrap_fn(ds, to_height, \"from_height\", from_name)\n", - " wnd_spd.attrs[\"long_name\"] = (\n", - " wnd_spd.attrs[\"long_name\"] + \", \" + f\"from nearest height = {from_height}\"\n", - " )\n", - "\n", - " return wnd_spd.rename(to_name)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "geodata", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/source/conf.py b/docs/source/conf.py index ccd14f5d..ea9d69e7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ from geodata import __version__ project = "Geodata" -copyright = "2023, Geodata Contributors" +copyright = "2025, Geodata Contributors" author = "Geodata Contributors" release = __version__ diff --git a/docs/source/datasets/era5.rst b/docs/source/datasets/era5.rst new file mode 100644 index 00000000..2af0aee6 --- /dev/null +++ b/docs/source/datasets/era5.rst @@ -0,0 +1,81 @@ +ERA5 Specific Instructions +========================== + +This page explains how you can set up access to ERA5 data from the `Copernicus Data Store `_. + +Creating a CDS account +---------------------- + +To download ERA5 data from the CDS, you'll need to create a free `CDS account here `_. + +Download data through CDS API +----------------------------- + +Once your account has been created, set up access to the API by following these steps: + +1. Log into your CDS account and visit your `profile page `_. +2. Install the API key. There will be a section called **Personal Access Token**. + Copy these two lines into a file called ``.cdsapirc`` in your user root folder. + +- **macOS/Linux**: Open a terminal and run: + + .. code-block:: bash + + touch ~/.cdsapirc + + Then add the lines using: + + .. code-block:: bash + + echo [line 1 of the code] >> ~/.cdsapirc + echo [line 2 of the code] >> ~/.cdsapirc + + + - **Windows**: The process is slightly more complicated. Please refer to the in-depth guide at the Copernicus Knowledge Base `here `_. + +3. Install the CDS API client by opening a terminal/shell and running + +.. code-block:: bash + + pip install ".[download]" + +(Assuming you are in Geodata's *root directory*.) + +1. Once you've installed the API key and the API client, confirm access by running an + example in a Python script or a Jupyter notebook: + +.. code-block:: python + + import cdsapi + + c = cdsapi.Client() + + c.retrieve( + "reanalysis-era5-single-levels", + { + "product_type": "reanalysis", + "format": "netcdf", + "variable": [ + "2m_dewpoint_temperature", + "2m_temperature", + ], + "year": "2011", + "month": [ + "01", + ], + "day": ["01", "02", "03"], + "time": [ + "00:00", + "12:00", + ], + }, + "download.nc", + ) + +The above example downloads 2m temperature and 2m dewpoint temperature with data points +at 00:00 and 12:00 for each day, from January 1-3, 2011, in NetCDF format. + +If this works, you have successfully set up access to the ERA5 data through the CDS API. +Please subsequently refer to the `general documentation on datasets <../overview.rst>`_ +for more information on how to download ERA5-based datasets using the ``geodata`` +package. diff --git a/docs/source/datasets/era5/era5.ipynb b/docs/source/datasets/era5/era5.ipynb deleted file mode 100644 index 7d8d0696..00000000 --- a/docs/source/datasets/era5/era5.ipynb +++ /dev/null @@ -1,580 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ERA5 Analysis Process\n", - "\n", - "This Jupyter notebook provides a brief overview of how to use the **geodata** package to download ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview), create geographic-temporal subsets called cutouts, and use those cutouts to generate standalone datasets for separate analysis.\n", - "\n", - "*The following guide assumes you have installed and configured **geodata** and all required dependencies.*" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1 - Setup\n", - "\n", - "Import the package first." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import geodata" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notifications in **geodata** are implemented using `loggers` from the `logging` library.\n", - "It is recommended to always launch a logger to get information on what is going on. For debugging, you can use the more verbose `level=logging.DEBUG`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "logging.basicConfig(level=logging.INFO)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2 - Download and Create Cutout\n", - "Assuming you have previously created a CDS account and set up the CDS API credentials, you can download ERA5 data from the CDS API as follows.\n", - "\n", - "First, define a dataset object for the data you wish to download:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## For ERA5, pass geographic bounds in array as follows:\n", - "## bounds = [North, West, South, East]\n", - "## Omitting bounds will default to global file of 20+ GB per month\n", - "DS = geodata.Dataset(\n", - " module=\"era5\",\n", - " weather_data_config=\"wind_solar_hourly\",\n", - " years=slice(2005, 2005),\n", - " months=slice(1, 2),\n", - " bounds=[50, -3, 45, 3],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* Use `module` to specify the data source. In this example, it is \"era5\".\n", - "* Use `weather_data_config` to specifiy the dataset. In this example, hourly data is used, as specified by the `\"wind_solar_hourly\"` value.\n", - "* Use `years=slice()` and `months=slice()` to specify the years and months for download. In each parameter, the first value indicates the start period, and the second value the end period.\n", - "* Use `bounds` to specify the geographic bounds to which you wish to limit your download data. `bounds` should be set as follows: `bounds = [North, West, South, East]`. Omitting bounds will default to downloading a global file of 20+ GB per month.\n", - "\n", - "Use the code block below to begin the download." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When a `dataset` object is created, **geodata** performs a check to see if the data specified has already been downloaded by checking for the existence of ERA5 datafiles in the `era5` directory configured in `src/geodata/config.py` (downloaded data is placed into subdirectories by year and then - for daily files - by month, ie `2011/01, 2011/02, 2012/01`, etc). Monthly files are simply placed in the month's folder. If downloaded data is found, the `prepared` attribute is set to `True` upon `dataset` object declaration.\n", - "\n", - "Accordingly, the snippet below saves you the trouble of accidentally redownloading data if it is already present in the correct subdirectories." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if DS.prepared == False:\n", - " DS.get_data()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, in order to use the downloaded ERA5 data with **geodata**, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "DS.trim_variables()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`trim_variables()` subsets and resaves the downloaded files so that only those variables needed to generate **geodata** outputs are kept." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3 - Create Cutout\n", - "\n", - "A cutout is a subset of downloaded data based on specified time periods and geographic coordinates. Cutouts are saved to the cutout directory specified in `src/geodata/config.py` and can be used to generate multiple outputs.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout = geodata.Cutout(\n", - " name=\"era5-europe-test-2011-02\",\n", - " module=\"era5\",\n", - " weather_data_config=\"wind_solar_hourly\",\n", - " xs=slice(1, 2),\n", - " ys=slice(48, 46),\n", - " years=slice(2005, 2005),\n", - " months=slice(1, 1),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The above code creates a cutout for January 2011 for a geographic area corresponding to a portion of Europe. Walking through the parameters:\n", - "\n", - "* `name` will be the name of the directory created in the cutouts folder where **geodata** will place the data files corresponding to the cutout.\n", - "* `module` indicates the source for the data from which the cutout is created.\n", - "* Use `xs=slice()` and `ys=slice()` to define a geographical range for the cutout. These para\n", - "* Use `years=slice()` and `months=slice()` to define a temporal range for the cutout. \n", - "\n", - "`geodata.Cutout()` only defines the cutout object in memory. To actually create the cutout files, run `prepare()`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.prepare()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Running `cutout.prepare()` as above will create the cutout by downloading and then subsetting the ERA5 data. Accordingly, the above code block could take a while to finish processing.\n", - "\n", - "`prepare()` will first perform a check to see if a cutout has already been created at the specified directory, and will exit the download. creation process if a cutout already exists. To override this behavior and force a redownload and recalculation of the cutout, run `prepare(overwrite=True)`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To verify the results of the cutout, you can print some attributes to the console as follows.\n", - "\n", - "Basic information:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Name:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Coordinates:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.coords" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "All metadata:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cutout.meta" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4 - Generate Outputs\n", - "\n", - "**geodata** currently supports the following outputs using ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview).\n", - "\n", - "### Wind\n", - "* Wind generation time-series (`wind`)\n", - "* Wind speed time-series (`windspd`)\n", - "\n", - "### Solar\n", - "* Solar photovoltaic generation time-series (`pv`)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Wind Generation Time-series\n", - "Convert wind speeds for turbine to wind energy generation using the following code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_wind = geodata.convert.wind(cutout, turbine=\"Suzlon_S82_1.5_MW\", smooth=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `turbine` - **string or dict** - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of Turbines here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/windturbine)\n", - "* `smooth` - **bool or dict** - If True smooth power curve with a gaussian kernel as determined for the Danish wind fleet to Delta_v = 1.27 and sigma = 2.29. A dict allows to tune these values.\n", - "\n", - "*Note* - \n", - "You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_wind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind = ds_wind.to_dataframe(name=\"wind\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_wind.to_csv(\"era5_wind_data.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Wind Speed Density Time-series\n", - "Extract wind speeds at given height (ms-1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_windspd = geodata.convert.windspd(cutout, turbine=\"Vestas_V66_1750kW\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `**params` - Must have 1 of the following:\n", - " - `turbine` - **string or dict** - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of Turbines here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/windturbine)\n", - " - `hub-height` - **num** - Extrapolation height (m)\n", - " \n", - "*Note* - \n", - "You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_windspd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd = ds_windspd.to_dataframe(name=\"windspd\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_windspd.to_csv(\"era_windspd_data.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Solar Photovoltaic Generation Time-series\n", - "\n", - "Convert downward-shortwave, upward-shortwave radiation flux and ambient temperature into a pv generation time-series.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_pv = geodata.convert.pv(cutout, panel=\"KANEKA\", orientation=\"latitude_optimal\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Going over the parameters:\n", - "\n", - "* `cutout` - **string** - A cutout created by `geodata.Cutout()`\n", - "* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel)\n", - "* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions.\n", - "* (optional) clearsky_model - string or None - \tEither the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_pv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To convert this array to a more conventional dataframe, run:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv = ds_pv.to_dataframe(name=\"pv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which converts the xarray dataset into a pandas dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To output the data to a csv for separate analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_pv.to_csv(\"era_pv_data.csv\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/source/datasets/era5/era5_download.md b/docs/source/datasets/era5/era5_download.md deleted file mode 100644 index 9f71c48a..00000000 --- a/docs/source/datasets/era5/era5_download.md +++ /dev/null @@ -1,217 +0,0 @@ -# Download ERA5 Dataset and Create ERA5 Cutouts - -A short guide to using **geodata** to downloading and creating cutouts from ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/). - -## Download Dataset - -Download methods for ERA5 data are built into the **geodata** package. A basic example is as follows: - - -To start, import the required dependencies: - -```python -import geodata - -dataset = geodata.Dataset( - module="era5", - weather_data_config="wind_solar_hourly", - years=slice(2005, 2005), - months=slice(1, 2), - bounds=[50, -3, 45, 3], -) - -if not dataset.prepared: - dataset.get_data() - -dataset.trim_variables(downloadedfiles=True) -``` - -Let's breakdown the code above: - -```python -import geodata -``` -Importing the logging package allows **geodata** to generate console input for download status, errors, etc. -Importing geodata is necessary to run **geodata**. - -```python -dataset = geodata.Dataset( - module="era5", - weather_data_config="wind_solar_hourly", - years=slice(2005, 2005), - months=slice(1, 2), - bounds=[50, -3, 45, 3], -) -``` - -`geodata.Dataset()` creates a dataset object via which you can download data files. To create objects for ERA5 data, specify `module="era5"`. You must also have configured `era5_dir` in `config.py` to point to a directory on your local machine (to set up `config.py`, [see here](../../quick_start/packagesetup.md)). - -The `years` and `months` parameters allow you to specify start years/months and end years/months for the data download. The above example would download data for January to February 2005. Ranges based on more granular time periods (such as day or hour) are not currently supported, but may be available in a future release. - -Running the above code does not actually download the data yet. Instead, it checks whether the indicated files for download are present in the local directory specified in `config.py`: - -``` ->> INFO:geodata.dataset:Directory /home/user/.local/geodata/era5 found, checking for completeness. ->> INFO:geodata.dataset:Directory complete. -``` - -and returns a `dataset` object indicating whether the data is "prepared." - -``` - -``` - -A "prepared" dataset indicates that the directories for storing the data - which take the form `/era5/{years}/{months}` for every unique time period in the data - have been created and are populated with downloaded data. - -```python -if not dataset.prepared: - dataset.get_data() -``` -If the dataset is "Unprepared", this conditional statement will download the actual netcdf files from the ERA5 CDS API. - - -```python -dataset.trim_variables(downloadedfiles = True) -``` -To save hard disk space, **geodata** allows you to trim the downloaded datasets to just the variables needed for the package's supported outputs. - -## Create Cutout - -A cutout is the basis for any data or analysis output by the **geodata** package. Cutouts are stored in the directory `cutout_dir` configured in `config.py` (to set up `config.py`, [see here](../../quick_start/packagesetup.md)). - -```python -cutout = geodata.Cutout( - name="era5-europe-test-2011-01", - module="era5", - weather_data_config="era5_monthly", - xs=slice(30, 41.56244222), - ys=slice(33.56459975, 35), - years=slice(2011, 2011), - months=slice(1, 1), -) -``` - -To prepare a cutout, the following must be specified for `geodata.Cutout()`: - -* The cutout name -* The source dataset -* Time range -* Geographic range as represented by `x` and `y` coordinates. - -The example in the code block above uses ERA5 data, as specified by the `module=era5` parameter. - -`xs` and `ys` in combination with the `slice()` function allow us to specify a geographic range based on longitude and latitude. The above example subsets a portion of Europe. - -`years` and `months` are used to subset the time range. For both functions, the first value represents the start point, and the second value represents the end point. The above example creates a cutout for January 2011. - -## Prepare Cutout - -To create a cutout, we can run `cutout.prepare()`. The **geodata** package will create a folder in the cutout directory you specified in `config.py` with the name specified in `geodata.Cutout()` (in the above example, `era5-europe-test-2011-01`). The folder, depending on the date range, will then contain one or more monthly netcdf files containing ERA5 data corresponding to the temporal and geographical ranges indicated when the cutout was created. Data files in the cutout folder will be at the monthly level - i.e., there will be one file for each month in the specified download time range. - -To actually create the cutout, you must run `cutout.prepare()`. Upon running `cutout.prepare()`, **geodata** will check for the presence of the cutout and abort if the cutout already exists. If you want to force the regeneration of a cutout, run the command with the parameter `overwrite=True`. - - -## Cutout Metadata - -You can query various metadata associated with a cutout. Querying returns the name, geographic and time range, and the preparation status of the cutout (i.e., whether cutout.prepare() has been run, creating the .nc files making up the cutout data). - -```python - -``` - -- `cutout.name` returns just the name: - -``` -'era5-europe-test-2011-01' -``` - -- `cutout.coords` returns coordinates: -```python -Coordinates: - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 - lon (x) float32 ... - lat (y) float32 ... - * year-month (year-month) MultiIndex - - year (year-month) int64 2011 - - month (year-month) int64 1 -``` - -- `cutout.meta` returns all associated metadata: -```python - -Dimensions: (time: 744, x: 47, y: 5, year-month: 1) -Coordinates: - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 - lon (x) float32 ... - lat (y) float32 ... - * year-month (year-month) MultiIndex - - year (year-month) int64 2011 - - month (year-month) int64 1 -Data variables: - height (y, x) float32 ... -Attributes: - Conventions: CF-1.6 - history: 2020-03-14 04:29:35 GMT by grib_to_netcdf-2.16.0: /opt/ecmw... - module: era5 - view: {'x': slice(30, 41.56244222, None), 'y': slice(35, 33.56459... -``` - -To understand the variables that were downloaded, you can run: -``` -cutout.dataset_module.weather_data_config -``` - -Which will return the selected download settings for the ERA5 data. - - -```python -{'wind_solar_hourly': {'api_func': , - 'file_granularity': 'monthly', - 'tasks_func': , - 'meta_prepare_func': , - 'prepare_func': , - 'template': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_hourly.nc', - 'fn': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_hourly.nc', - 'product': 'reanalysis-era5-single-levels', - 'product_type': 'reanalysis', - 'variables': ['100m_u_component_of_wind', - '100m_v_component_of_wind', - '2m_temperature', - 'runoff', - 'soil_temperature_level_4', - 'surface_net_solar_radiation', - 'surface_pressure', - 'surface_solar_radiation_downwards', - 'toa_incident_solar_radiation', - 'total_sky_direct_solar_radiation_at_surface', - 'forecast_surface_roughness', - 'orography'], - 'meta_attrs': {'Conventions': 'CF-1.6', - 'history': '2022-01-11 00:43:10 GMT by grib_to_netcdf-2.23.0: /opt/ecmwf/mars-client/bin/grib_to_netcdf -S param -o /cache/data3/adaptor.mars.internal-1641861772.7584445-3425-17-8ff8bcdc-2c08-4aed-95db-e0337692a384.nc /cache/tmp/8ff8bcdc-2c08-4aed-95db-e0337692a384-adaptor.mars.internal-1641861207.3577378-3425-30-tmp.grib', - 'module': 'era5'}}, - 'wind_solar_monthly': {'api_func': , - 'file_granularity': 'monthly', - 'tasks_func': , - 'meta_prepare_func': , - 'prepare_func': , - 'template': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_monthly.nc', - 'fn': '/Users/johndoe/data_for_geodata/era5/{year}/{month:0>2}/wind_solar_monthly.nc', - 'product': 'reanalysis-era5-single-levels-monthly-means', - 'product_type': 'monthly_averaged_reanalysis', - 'variables': ['100m_u_component_of_wind', - '100m_v_component_of_wind', - '2m_temperature', - 'runoff', - 'soil_temperature_level_4', - 'surface_net_solar_radiation', - 'surface_pressure', - 'surface_solar_radiation_downwards', - 'toa_incident_solar_radiation', - 'total_sky_direct_solar_radiation_at_surface', - 'forecast_surface_roughness', - 'orography']}} -``` diff --git a/docs/source/datasets/era5/era5_outputs.md b/docs/source/datasets/era5/era5_outputs.md deleted file mode 100644 index 5bf321e4..00000000 --- a/docs/source/datasets/era5/era5_outputs.md +++ /dev/null @@ -1,152 +0,0 @@ -# Generating Outputs with ERA5 Data - - - -**geodata** currently supports the following wind outputs using ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - -* Wind generation time-series (`wind`) -* Wind speed time-series (`windspd`) -* Solar photovoltaic generation time-series (`pv`) - -## Supported ERA5 Outputs - -### Wind Generation Time-series - -Convert wind speeds for turbine to wind energy generation. - -```python -cutout.wind(turbine: str | dict[str, str], smooth: bool = False) -``` - -#### Parameters - -* `cutout` - `Cutout` - A cutout created by `geodata.Cutout()` -* `turbine` - `str | dict` - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, see [the list of turbines here.](https://github.com/GeodataTools/geodata/tree/master/src/geodata/resources/windturbine) -* `smooth` - `bool | dict` - If `True`, smooth power curve with a gaussian kernel as determined for the Danish wind fleet to Delta_v = 1.27 and sigma = 2.29. A dict allows to tune these values. - -*Note*: You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`). - -#### Example Code - -```python -ds_wind = cutout.wind(turbine="Suzlon_S82_1.5_MW", smooth=True) -ds_wind.to_dataframe(name="wind") -``` - -### Wind Speed Time-series - -Extract wind speeds at given height (ms-1) - - -``` -geodata.convert.windspd(cutout: geodata.Cutout, **params) -``` - -#### Parameters - -* `cutout` - `str` - A cutout created by `geodata.Cutout()` -* `**params` - Must have 1 of the following: - - `turbine` - `str | dict` - Name of a turbine known by the reatlas client or a turbineconfig dictionary with the keys 'hub_height' for the hub height and 'V', 'POW' defining the power curve. For a full list of currently supported turbines, [the list of turbines here.](https://github.com/GeodataTools/geodata/tree/master/src/geodata/resources/windturbine) - - `hub-height` - `int` - Extrapolation height (m) -*Note*: You can also specify all of the general conversion arguments documented in the `convert_and_aggregate` function (e.g. `var_height='lml'`). -#### Example Code - -```python -ds_windspd = cutout.windspd(turbine='Vestas_V66_1750kW') -ds_windspd.to_dataframe(name = 'windspd') -``` - - -### Solar photovoltaic generation time-series - -Convert downward-shortwave, upward-shortwave radiation flux and ambient temperature into a pv generation time-series. - -```python -cutout.pv(panel, orientation, clearsky_model) -``` - -#### Parameters - -* `cutout` - **string** - A cutout created by `geodata.Cutout()` -* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel) -* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions. -* (optional) clearsky_model - string or None - Either the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity. - -#### Example Code and Result - -```python -ds_pv = geodata.convert.pv(panel="KANEKA", orientation = "latitude_optimal") -ds_pv.to_dataframe(name = 'pv') -``` - -## Example Output - -With the full list of supported ERA5 outputs above, we can see an example of generating output ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - - -### Setup - -Let's assume we've created an ERA5 cutout along the following lines: - -```python -cutout = geodata.Cutout(name="era5-europe-example", - module="era5", - weather_data_config="era5_monthly", - xs=slice(30, 41.56244222), - ys=slice(33.56459975, 35), - years=slice(2011, 2011), - months=slice(1,1) -) -``` - -Note that unlike datasets, we don't need explicitly prepare a cutout. Geodata will prepare the cutout automatically if it's not ready. We can now use this cutout to generate datasets. - -### Creating a Solar photovoltaic (pv) generation time-series - -To create a pv generation time-series, we can use the following code with our ERA5 cutout: - - -```python -ds = cutout.pv(panel="KANEKA", orientation="latitude_optimal") -``` - -Some information about the parameters: -* `panel` - string - Specify a solar panel type on which to base the calculation. **geodata** contains an internal solar panel dictionary with keys defining several solar panel characteristics used for the time-series calculation. For a complete list of included panel types, see [the list of panel types here.](https://github.com/east-winds/geodata/tree/master/geodata/resources/solarpanel) -* `orientation` - str, dict or callback - Panel orientation can be chosen from either `latitude_optimal`, a constant orientation such as `{'slope': 0.0,'azimuth': 0.0}`, or a callback function with the same signature as the callbacks generated by the `geodata.pv.orientation.make_*` functions. -* (optional) clearsky_model - string or None - Either the `simple` or the `enhanced` Reindl clearsky model. The default choice of None will choose dependending on data availability, since the `enhanced` model also incorporates ambient air temperature and relative humidity. - - -The convert function returns an xarray dataset, which is an in-memory representation of a NetCDF file: - -``` - -array([[[0., 0., 0., ..., 0., 0., 0.], - [0., 0., 0., ..., 0., 0., 0.], - [0., 0., 0., ..., 0., 0., 0.], - -Coordinates: - lon (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - lat (y) float32 34.815 34.565 34.315 34.065 33.815 - * x (x) float32 30.0 30.25 30.5 30.75 31.0 ... 40.75 41.0 41.25 41.5 - * y (y) float32 34.815 34.565 34.315 34.065 33.815 - * time (time) datetime64[ns] 2011-01-01 ... 2011-01-31T23:00:00 -``` - -To convert this array to a more conventional dataframe, we can run: - -```python -df = ds.to_dataframe(name='pv') -``` - -which converts the xarray dataset into a pandas dataframe. - - -The result is a dataset with an observation for each time period (in this ERA5 case, hourly) and geographic point with the calculated pv generation for each observation. - -Finally, we can run something like: - -```python -df.to_csv('era5_pv_data.csv') -``` - -to output the data to csv for use in other applications. diff --git a/docs/source/datasets/era5/index.md b/docs/source/datasets/era5/index.md deleted file mode 100644 index 462b826b..00000000 --- a/docs/source/datasets/era5/index.md +++ /dev/null @@ -1,58 +0,0 @@ -# ERA5 Related Tutorials - -This page explains how you can setup access to ERA5 data from the [Copernicus Data Store](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview). - -## Creating a CDS account - -To download the ERA5 data from the CDS, you'll need to create a free [CDS account here](https://cds.climate.copernicus.eu/). - -## Download data through CDS API -Once your account has been created, you'll need to set up access to the API by doing the following: - -1. Log into your CDS account and visit [your profile page](https://cds.climate.copernicus.eu/profile). -2. Install the API key. There will be a section called **Personal Access Token**. You will then need to copy these two lines this into - a file called _.cdsapirc_ that you will create in your user root folder. - - *macOS/Linux*: For macOS or Linux, you can create _.cdsapirc_ by opening a terminal window and running `touch ~/.cdsapirc` to create the file and then `echo [line 1 of the code] >> ~/.cdsapirc` followed by `echo [line 2 of the code] >> ~/.cdsapirc`. - - *Windows*: For Windows, the process is slightly more complicated. Please refer to an in-depth guide at the Copernicus Knowledge Base [here](https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+Windows). -3. Install the CDS API client by opening a terminal/shell and running `pip install ".[download]"`, assuming you are in Geodata's *root directory*. -4. Once you've installed the API key and the API client, you can confirm access by running an example in a Python script or a Jupyter notebook. - -```python -import cdsapi - -c = cdsapi.Client() - -c.retrieve( - "reanalysis-era5-single-levels", - { - "product_type": "reanalysis", - "format": "netcdf", - "variable": [ - "2m_dewpoint_temperature", - "2m_temperature", - ], - "year": "2011", - "month": [ - "01", - ], - "day": ["01", "02", "03"], - "time": [ - "00:00", - "12:00", - ], - }, - "download.nc", -) -``` - -The above example downloads 2m temperature and 2m dewpoint temperature with data points at 00:00 and 12:00 for each day, from January 1-3, 2011, in NetCDF format. - -## More Resources on ERA5 - -We've also provide a few tutorials on how to interact with the ERA5 datasets using Geodata. They can be found in the links below. - -```{toctree} -:maxdepth: 2 -:glob: -* -``` diff --git a/docs/source/datasets/merra2/index.md b/docs/source/datasets/merra2/index.md index 8e818eb3..fb020ff4 100644 --- a/docs/source/datasets/merra2/index.md +++ b/docs/source/datasets/merra2/index.md @@ -21,7 +21,7 @@ The following procedure is sourced from [GES DISC's documentation](https://disc. ## Configure API Crendentials -To download MERRA2 data via **geodata** you'll need to install the API credentials locally. +To download MERRA2 data via **geodata** you'll need to install the API credentials locally. ### macOS/Linux @@ -36,17 +36,14 @@ where `[login]` is your Earthdata user name and `[password]` is your Earthdata L ### Windows For Windows, open Notepad and enter the following line in a new document, making sure to substitute `` and ``for your Earthdata login credentials: - + `machine urs.earthdata.nasa.gov login password ` Save the file to `C:\Users\\.netrc` ## What' next? -We provide a few more tutorials on how to download and utilize the MERRA2 dataset with Geodata. You can find them below. - -```{toctree} -:maxdepth: 2 -:glob: -* -``` +Now that you have configured your Earthdata Login credentials, you have successfully set up access to the MERRA-2 data. +Please subsequently refer to the [general documentation on datasets](../overview.rst) +for more information on how to download ERA5-based datasets using the `geodata` +package. diff --git a/docs/source/datasets/overview.rst b/docs/source/datasets/overview.rst new file mode 100644 index 00000000..57430059 --- /dev/null +++ b/docs/source/datasets/overview.rst @@ -0,0 +1,93 @@ +========================== +Dataset Module Overview +========================== + +The ``geodata.datasets`` module provides tools and classes for accessing, managing, and +processing geospatial datasets. It offers a unified interface for loading various +data formats, handling metadata, and performing common geospatial operations. + +Key Features +------------ + +- Supports the download and management of datasets from various sources, such as + `ERA5 `_ and + `MERRA2 `_. + +- Provides a consistent API for accessing geospatial data, regardless of the underlying + data source. + +Typical Usage +------------- + +In the following example, we will demonstrate how to download a dataset containing wind +and solar data from ECMWF's ERA5 dataset. + +.. code-block:: python + + from geodata.datasets import load_dataset + + dataset_cls = load_dataset("wind_solar_hourly") + + years = slice(2010, 2020) + months = slice(1, 13) + dataset = dataset_cls(years=years, months=months) + +Here, we first create a dataset class using the `load_dataset` function, specifying the +name of the dataset we want to load. We then instantiate the dataset class with the +desired time range (years and months). Then, we can create a dataset instance with +that class, which will handle the downloading and processing of the data. + +Dataset Classes +----------------- +The `geodata.datasets` module includes several dataset classes, each tailored for +specific datasets. These classes encapsulate the logic for downloading, processing, and +accessing the data. Some of the available dataset classes +(listed by `weather_data_config`) include: + +- `wind_solar_hourly`: A dataset containing hourly wind and solar data from ECMWF's + ERA5. It is important to note that the wind data are only recorded at + 10 and 100 meters above ground level. Hence, this dataset is also referred to as + 2D wind and solar dataset. + +- `wind_3d_hourly`: A dataset containing hourly wind data from ECMWF's ERA5 at + multiple vertical levels, providing a more comprehensive view of the wind profile. + It can be used for wind speed estimation using and interpolation model built into + the geodata library. + +You can use the `list_datasets` function to see all available datasets in the +`geodata.datasets` module. This function returns a list of dataset names that can be +loaded using the `load_dataset` function. For example: + +.. code-block:: python + + from geodata.datasets import list_datasets + + available_datasets = list_datasets() + print(available_datasets) # Outputs a list of available dataset names. + +Check Preparedness of Datasets +------------------------------------------------ +To check if a dataset is prepared and ready for use, you can use the `downloaded` +property of the dataset instance. This property returns a boolean indicating whether the +dataset is fully prepared. If the dataset is not prepared, you can call the `prepare` +method to download and process the data. For example: + +.. code-block:: python + + print(dataset.downloaded) # Check if the dataset is downloaded. Outputs False here. + + if not dataset.downloaded: + dataset.download() + + print(dataset.downloaded) # Outputs True after downloading. + +Dataset's Interoperability with Cutout +------------------------------------------------ + +At the moment, the dataset classes are not interoperable with the `Cutout` class. +In the future, we plan to consolidate the functionalities of the `Cutout` class into the +dataset classes and the modeling module (see :doc:`here<../modeling/wind/index>`). + +For now, after downloading a dataset, a good point to move forward would be to use the +:doc:`modeling module <../modeling/index>` to create a model that can do certain types +of modeling with the dataset, such as wind speed estimation or solar PV generation. diff --git a/docs/source/index.rst b/docs/source/index.rst index 4674cc80..fc398a3b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -26,6 +26,15 @@ Welcome to Geodata's documentation! datasets/merra2/index datasets/* +.. toctree:: + :maxdepth: 1 + :caption: Modeling + :glob: + :hidden: + + modeling/wind/index + modeling/* + .. toctree:: :maxdepth: 1 :caption: Mask @@ -57,15 +66,6 @@ Welcome to Geodata's documentation! .. application/* -.. toctree:: - :maxdepth: 1 - :caption: Modeling - :glob: - :hidden: - - modeling/wind/index - modeling/* - .. toctree:: :maxdepth: 1 :caption: API Reference diff --git a/docs/source/modeling/wind/extrapolation.rst b/docs/source/modeling/wind/extrapolation.rst index af92a2b4..a983806e 100644 --- a/docs/source/modeling/wind/extrapolation.rst +++ b/docs/source/modeling/wind/extrapolation.rst @@ -1,7 +1,8 @@ Tutorial: Estimate Wind Speed with Extrapolation ================================================ -In this tutorial, we will learn how to estimate wind speed using the extrapolation model from the geodata library. +In this tutorial, we will learn how to estimate wind speed using the extrapolation model + from the geodata library. .. warning:: Performing wind speed estimation using extrapolation requires a dataset with known @@ -20,11 +21,12 @@ To get started, we need to import the required libraries. We will import the `Wi .. code:: Python - import geodata import xarray as xr + from geodata.datasets import load_dataset from geodata.model.wind import WindExtrapolationModel + Step 2: Load the dataset ------------------------ @@ -33,20 +35,25 @@ Next, we need to load the dataset that contains the wind speed data. We will use .. code:: Python # Load the dataset - ds = geodata.Dataset( - module="merra2", - weather_data_config="slv_flux_hourly", - years=slice(2010, 2010), + ds_cls = load_dataset("slv_flux_hourly") + ds = ds_cls( + years=slice(2006, 2006), months=slice(1, 1), - bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box ) - if not ds.prepared: - ds.get_data() # Download the data if we don't have it locally + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + Step 3: Compute extrapolation parameters -------------------------------------------- -The extrapolation is separated into two steps, first estimating extrapolation parameters using linear regression, and second extrapolating to desired heights). First, we compute the extrapolation parameters. For more information on the model, see below "How the Extrapolation Model Works". +The extrapolation is separated into two steps, first estimating extrapolation parameters +using linear regression, and second extrapolating to desired heights. +First, we compute the extrapolation parameters. +For more information on the model, see the section below: `How the Extrapolation Model Works`_. .. code:: Python @@ -57,7 +64,8 @@ The extrapolation is separated into two steps, first estimating extrapolation pa model = WindExtrapolationModel(ds) model.prepare() -If you have already prepared a cutout with the config :code:`slv_flux_hourly`, you can also pass +If you have already prepared a cutout with the config :code:`slv_flux_hourly`, you +can also pass that into the model as well. The model treats dataset and cutouts indifferently. Simply replace :code:`ds` with your cutout variable. @@ -72,11 +80,11 @@ Simply replace :code:`ds` with your cutout variable. point on, you can load and use the model directly without re-preparing it. Step 4: Estimate using the extrapolation model ----------------------------------- +---------------------------------------------- Now that we have prepared the model, we can perform the extrapolation to estimate wind speed at the desired locations. Suppose we want to estimate the wind speed at a height -of 60 above ground during Jaunary of 2006 for the entire region covered by the original +of 60 above ground during January of 2006 for the entire region covered by the original dataset, we can do this as follows: .. code:: Python @@ -91,13 +99,40 @@ This will return an xarray DataArray containing the estimated wind speed values. that you can also select a subset area by passing in :code:`xs=slice(start, end)` and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. -.. note:: - As the underlying MERRA2 dataset already contained wind speed at certain heights, the - model also has a feature to return the original wind speed values from the dataset - if desired. To do this, simply set the `use_real_data` parameter to `True` in the - `estimate` method. You do not need worry about whether the height you queried is - available in the dataset; the model will handle that for you. If the height is not - available, it will perform extrapolation instead. + +Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model +-------------------------------------------------------------------------------- + +Geodata also supports a limited set of wind turbine models to estimate the capacity +factor (CF) of a wind turbine directly. To get a list of available wind turbine models, +you can use the `get_available_windturbines` function: + +.. code:: Python + + from geodata.resource import get_available_windturbines + + turbines = get_available_windturbines() + print(turbines) # List of available wind turbine configurations + + +To estimate the capacity factor of a wind turbine, you can use the `estimate` method +and passign in the `turbine` parameter with the name of the wind turbine model. + +.. code:: Python + + # Estimate the capacity factor for a specific wind turbine model + estimated_cf: xr.Dataset = model.estimate( + turbine="Vestas_V112_3MW", # Example wind turbine model + years=slice(2006, 2006), + months=slice(1, 1), + ) + + print(estimated_cf) # Display the estimated capacity factor + + +The output will be an xarray Dataset containing the estimated capacity factor values +for the specified wind turbine model over the given time period and region. + How the Extrapolation Model Works --------------------------------- diff --git a/docs/source/modeling/wind/index.rst b/docs/source/modeling/wind/index.rst index 234ba7ca..e6a93019 100644 --- a/docs/source/modeling/wind/index.rst +++ b/docs/source/modeling/wind/index.rst @@ -8,8 +8,9 @@ estimate wind speed with two modes: interpolation and extrapolation. How to use the models --------------------- -Unlike other modules of geodata, the model module does not get imported automatically. -In other words, one cannot use the models with an import statement like this: +Unlike some other modules of geodata, the model module does not get imported +automatically. In other words, one cannot use the models with an import statement +like this: .. code:: Python @@ -26,8 +27,54 @@ Instead, the user must import the models explicitly: The reason for this is to keep the main geodata namespace clean, since we might add many more models in the future. -More details on how to use the models can be found in each model's respective tutorial -as well as in the API reference. +In general, models are created based off of a dataset object, we must be downloaded +first. We'll use the `wind_3d_hourly` dataset from the ERA5 dataset as an example: + +.. code:: Python + + from geodata.datasets import load_dataset + from geodata.model.wind import WindInterpolationModel + + # Load the dataset + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( + years=slice(2006, 2006), + months=slice(1, 1), + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + ) + + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + + +Once we have the dataset, we can create a model based on it. +The model will be associated with the dataset forever, so if you wish to use a +different dataset, you will need to create a new model. + +.. code:: Python + + # Create a model based on the above dataset + model = WindInterpolationModel(ds) + model.prepare() # Prepare the model + + print(model.prepared) # Check if the model is prepared. Should return True. + + +Once the model is prepared, we can use it to estimate wind speed at desired heights. + +.. code:: Python + + # Estimate wind speed at a specific height (e.g., 100 meters) + wind_speed: xr.Dataset = model.estimate(height=100.0) + + # The wind_speed variable is an xarray Dataset containing the estimated wind speed at the specified height. + # Over the region covered by the original dataset. + + +The above demonstrates the typical workflow. More model-specific details can be found +in each model's respective tutorial as well as in the API reference. .. toctree:: :maxdepth: 1 diff --git a/docs/source/modeling/wind/interpolation.rst b/docs/source/modeling/wind/interpolation.rst index d946a1f4..788bba56 100644 --- a/docs/source/modeling/wind/interpolation.rst +++ b/docs/source/modeling/wind/interpolation.rst @@ -1,7 +1,8 @@ Tutorial: Estimate Wind Speed with Interpolation ================================================ -In this tutorial, we will learn how to estimate wind speed using the interpolation model from the geodata library. +In this tutorial, we will learn how to estimate wind speed using the interpolation model + from the geodata library. .. warning:: Performing wind speed estimation using interpolation requires a dataset with known @@ -16,37 +17,75 @@ In this tutorial, we will learn how to estimate wind speed using the interpolati Step 1: Import the necessary libraries ---------------------------------------- -To get started, we need to import the required libraries. We will import the `WindInterpolationModel` from the `geodata` library, as well as any other libraries needed for data handling and visualization. +To get started, we need to import the required libraries. We will import +the `WindInterpolationModel` from the `geodata` library, as well as any other +libraries needed for data handling and visualization. + .. code:: Python - import geodata import xarray as xr + from geodata.datasets import load_dataset from geodata.model.wind import WindInterpolationModel + Step 2: Load the dataset ------------------------ -Next, we need to load the dataset that contains the wind speed data. We will use the `wind_3d_hourly` dataset from the ERA5 dataset. +Next, we need to load the dataset that contains the wind speed data. +We will use the `wind_3d_hourly` dataset from the ERA5 dataset. + .. code:: Python # Load the dataset - ds = geodata.Dataset( - module="era5", - weather_data_config="wind_3d_hourly", + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( years=slice(2006, 2006), months=slice(1, 1), - bounds=[-10, 35, 10, 45] # Optional: specify the bounding box + bounds=[-10, 35, 10, 45] # Optional: specify the bounding box ) - if not ds.prepared: - ds.get_data() # Download the data if we don't have it locally + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + Step 3: Compute interpolation parameters -------------------------------------------- -The interpolation is separated into two steps to separate the computationally-intensive step (estimating interpolation parameters) from the computationally-easy step (interpolating at desired heights). First, we compute the interpolation parameters. +The interpolation is separated into two steps to separate the computationally-intensive +step (estimating interpolation parameters) from the computationally-easy step +(interpolating at desired heights). + +.. note:: + Wind-speed estimation using **cubic spline interpolation** fits a smooth, piecewise cubic polynomial + to known wind-speed data across spatial or temporal dimensions. Given a set of data points + :math:`(x_0, y_0), (x_1, y_1), \dots, (x_n, y_n)`, where :math:`x_i` are known positions (e.g., time or altitude) + and :math:`y_i` are corresponding wind speeds, the cubic spline for each interval + :math:`[x_i, x_{i+1}]` is defined as: + + .. math:: + + S_i(x) = a_i + b_i(x - x_i) + c_i(x - x_i)^2 + d_i(x - x_i)^3 + + The coefficients :math:`a_i, b_i, c_i, d_i` are determined by solving a system of equations subject to: + + 1. **Interpolation condition**: + :math:`S_i(x_i) = y_i`, and :math:`S_i(x_{i+1}) = y_{i+1}` + 2. **Continuity of first derivative**: + :math:`S_i'(x_{i+1}) = S_{i+1}'(x_{i+1})` + 3. **Continuity of second derivative**: + :math:`S_i''(x_{i+1}) = S_{i+1}''(x_{i+1})` + 4. **Boundary conditions**, typically *natural*: + :math:`S_0''(x_0) = 0`, and :math:`S_{n-1}''(x_n) = 0` + + The resulting spline provides a smooth and continuous estimate of wind speed, allowing accurate + interpolation between measured data points. With `wind_3d_hourly`, we have wind speeds at seven different heights AGL. + This enables us to estimate wind speeds at any height AGL within that range (from approximately 10m AGL to 170m AGL). + +First, we compute the interpolation parameters. .. code:: Python @@ -71,22 +110,56 @@ Simply replace :code:`ds` with your cutout variable. interpolation. You only need to call it once after loading the dataset. From that point on, you can load and use the model directly without re-preparing it. -Step 4: Estimate using the interpolation model ----------------------------------- +Step 4: Estimate wind speeds using the interpolation model +---------------------------------------------------------- Now that we have prepared the model, we can perform the interpolation to estimate wind speed at the desired locations. Suppose we want to estimate the wind speed at a height -of 60 m above ground during Jaunary of 2006 for the entire region covered by the original -dataset, we can do this as follows: +of 60 m above ground during January of 2006 for the entire region covered +by the original dataset, we can do this as follows: .. code:: Python - estimated_wind_speed = model.estimate( - height=60, + estimated_wind_speed: xr.Dataset = model.estimate( + height=60.0, years=slice(2006, 2006), months=slice(1, 1), ) -This will return an xarray DataArray containing the estimated wind speed values. Note + +This will return an xarray Dataset containing the estimated wind speed values. Note that you can also select a subset area by passing in :code:`xs=slice(start, end)` and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. + +Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model +-------------------------------------------------------------------------------- + +Geodata also supports a limited set of wind turbine models to estimate the capacity +factor (CF) of a wind turbine directly. To get a list of available wind turbine models, +you can use the `get_available_windturbines` function: + +.. code:: Python + + from geodata.resource import get_available_windturbines + + turbines = get_available_windturbines() + print(turbines) # List of available wind turbine configurations + + +To estimate the capacity factor of a wind turbine, you can use the `estimate` method +and passign in the `turbine` parameter with the name of the wind turbine model. + +.. code:: Python + + # Estimate the capacity factor for a specific wind turbine model + estimated_cf: xr.Dataset = model.estimate( + turbine="Vestas_V112_3MW", # Example wind turbine model + years=slice(2006, 2006), + months=slice(1, 1), + ) + + print(estimated_cf) # Display the estimated capacity factor + + +The output will be an xarray Dataset containing the estimated capacity factor values +for the specified wind turbine model over the given time period and region. diff --git a/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg b/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg deleted file mode 100644 index 1b3dc80d..00000000 --- a/docs/source/parameterizations/tex/084c4d336ce7b83477dc39e2638fa8c5.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg b/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg deleted file mode 100644 index 36f95779..00000000 --- a/docs/source/parameterizations/tex/142a0e55e7be4656275f11bc455886d4.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg b/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg deleted file mode 100644 index 5ea91e8f..00000000 --- a/docs/source/parameterizations/tex/191de8bb417c3c87a59bd9e2f4222ed0.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg b/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg deleted file mode 100644 index b152ece5..00000000 --- a/docs/source/parameterizations/tex/42c4a538b0898977509d72e9843ffaad.svg +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg b/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg deleted file mode 100644 index 38e293a5..00000000 --- a/docs/source/parameterizations/tex/46f5f4c88affca88279076c9fd0b8502.svg +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg b/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg deleted file mode 100644 index 11862566..00000000 --- a/docs/source/parameterizations/tex/5421d2d8ce8d3d6a7eb5c5e6c23527a5.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg b/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg deleted file mode 100644 index b2afdbcc..00000000 --- a/docs/source/parameterizations/tex/547e731741446703f67244e558fc0508.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg b/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg deleted file mode 100644 index e5f2c7be..00000000 --- a/docs/source/parameterizations/tex/7db7bb668a45f8c25263aa8d42c92f0e.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg b/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg deleted file mode 100644 index 7474165f..00000000 --- a/docs/source/parameterizations/tex/85b8a1a1ba5b0d126cb83dc1fc238bbc.svg +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg b/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg deleted file mode 100644 index bb9bebf7..00000000 --- a/docs/source/parameterizations/tex/acbc3da08b00b7616f45697dcc62d44d.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg b/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg deleted file mode 100644 index d1b43f3f..00000000 --- a/docs/source/parameterizations/tex/b215101f20292f1e8b0eb0ee28cf1d61.svg +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg b/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg deleted file mode 100644 index 657839b2..00000000 --- a/docs/source/parameterizations/tex/bd971e27e71d9bf4792553ddfe23d537.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg b/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg deleted file mode 100644 index ebad50a2..00000000 --- a/docs/source/parameterizations/tex/c39017bed62ee8b2a521779ae976ebe0.svg +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/source/parameterizations/wind_parameterizations.md b/docs/source/parameterizations/wind_parameterizations.md deleted file mode 100644 index 2f9399f7..00000000 --- a/docs/source/parameterizations/wind_parameterizations.md +++ /dev/null @@ -1,44 +0,0 @@ -# Wind Extrapolation and Stability Correction - -## Extrapolation - -Extrapolating wind speeds from measured (or modeled) heights to other heights must assume a model of the atmosphere. This is particularly important for studying wind power generation as turbine hub heights can be 50-120m above the ground. Standard approaches include the log law and power law. - -Geodata allows users to customize the wind extrapolation function through the `extrap_fn` parameter in the call to `wind.extrapolate_wind_speed()`. By default, the log law ratio (`extrap_fn = log_ratio`) is used. - - -## Stability correction - -Simple extrapolation routines assume neutral stability of the atmosphere, potentially ignoring highly stable or unstable atmospheric conditions that could alter the wind speed profile at certain hours of the day. - - -**`L` Obukhov length** - -Most stability correction functions rely on the Obukhov length, which can be calculated from available parameters [[1]](#references): - -$L= - \dfrac{u_{* }^{3}\overline{\theta_{v}}\rho_{a}C_p}{\kappa gH_{v0}}$ -$H_{v0}=H_{f}+0.61C_{p}\dfrac{\overline{\theta}}{L_{e}}H_{L}$ -$\overline{\theta}=\overline{T}\left(p_{0}/p\right)^{\kappa_{p}}$ - -where $u_{* }$ is the friction velocity, $\overline{\theta_{v}}$ is the virtual temperature, $\rho_a$ is the density, $C_p$ is the specific heat, $\kappa=0.4$, and $H_{v0}$ is the virtual heat flux in terms of sensible and latent heat fluxes (negative if directed upwards). - - -**$\psi_m$ Stability correction** - -Extrapolated wind speeds are "corrected" via a stability correction function $\psi_m$ according to: - -$u'(z) = u(z) - \psi_m(z/L)$ - -where $\psi_m(z/L)$ takes different forms in the literature [[2]](#references). - -Geodata has the following stability correction functions installed (in terms of the parameter $z/L$): -- `psi_linear`: linear for positive parameters (This function does not perform well for high $z/L$) -- `psi_linearexp`: piecewise linear-exponential -- `psi_linearexpconst`: piecewise linear-exponential with maximum constant correction, illustrated here for various constants: -![Stability Correction](../_static/images/wind_stability_corrections.png) - - -## References - -- [1] Rose, S., & Apt, J. (2016). Quantifying sources of uncertainty in reanalysis derived wind speed. Renewable Energy, 94, 157–165. https://doi.org/10.1016/j.renene.2016.03.028 -- [2] Sharan, M., & Aditi. (2009). Performance of various similarity functions for nondimensional wind and temperature profiles in the surface layer in stable conditions. Atmospheric Research, 94(2), 246–253. https://doi.org/10.1016/j.atmosres.2009.05.014 diff --git a/pyproject.toml b/pyproject.toml index 9173310d..bab31149 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + [project] name = "geodata-re" version = "0.1.0" @@ -16,17 +20,19 @@ dependencies = [ "netcdf4>=1.7.1.post2", "boto3==1.26.46", "toolz>=0.12.1", - "pyproj==3.6.1", "requests>=2.32.3", "matplotlib==3.9.2", "rasterio==1.4.0", "rioxarray==0.14.0", "shapely>=2.0.6", - "progressbar2>=4.5.0", "geopandas>=1.0.1", "pyyaml>=6.0.2", - "dask>=2024.9.0", + "dask[distributed]>=2024.9.0", "tqdm>=4.66.5", + "h5netcdf>=1.6.1", + "pvlib>=0.12.0", + "timezonefinder>=6.5.9", + "pyproj>=3.6.1", ] requires-python = ">=3.10" readme = "README.md" @@ -35,7 +41,8 @@ license = {text = "GPLv3"} [project.optional-dependencies] download = [ - "cdsapi>=0.7.3", + "cdsapi>=0.7.5", + "herbie-data>=2025.5.0", ] notebook = [ "notebook>=7.2.2", @@ -44,7 +51,11 @@ docs = [ "sphinx>=8.0.0", "myst-nb>=1.1.2", "sphinx-book-theme>=1.1.3", - "sphinx-autoapi==3.3.2" + "sphinx-autoapi==3.3.2", + "doc8>=1.1.2", +] +accelerate = [ + "numba>=0.61.0", ] [tool.uv] diff --git a/src/geodata/convert.py b/src/geodata/convert.py index f265b93f..2dfaf5c3 100644 --- a/src/geodata/convert.py +++ b/src/geodata/convert.py @@ -1,4 +1,5 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) +# Copyright 2025 Xiqiang Liu, Michael Davidson (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -21,132 +22,41 @@ import datetime as dt import logging from operator import itemgetter +from typing import TYPE_CHECKING, Literal, Callable import numpy as np import xarray as xr -from six import string_types -from tqdm.auto import tqdm from . import wind as windm from .pv.irradiation import TiltedIrradiation from .pv.orientation import SurfaceOrientation, get_orientation # noqa: F401 from .pv.solar_panel_model import SolarPanelModel from .pv.solar_position import SolarPosition +from .resource import ( + get_solarpanelconfig, + get_windturbineconfig, + windturbine_smooth, +) -logger = logging.getLogger(__name__) - - -def convert_cutout(cutout, convert_func, show_progress=False, **convert_kwds): - """ - Convert and aggregate a weather-based renewable generation time-series. - - NOTE: Not meant to be used by the user him or herself. Rather it is a - gateway function that is called by all the individual time-series - generation functions like pv and wind. Thus, all its parameters are also - available from these. - - Parameters (passed through as **params) - --------------------------------------- - show_progress : boolean|string - Whether to show a progress bar if boolean and its label if given as a - string (defaults to True). - - Returns - ------- - resource : xr.DataArray - Time-series of renewable generation aggregated to buses, if - `matrix` or equivalents are provided else the total sum of - generated energy. - - Internal Parameters (provided by f.ex. wind and pv) - --------------------------------------------------- - convert_func : Function - Callback like convert_wind, convert_pv - """ - if not cutout.prepared: - raise RuntimeError("The cutout has to be prepared first.") - - results = [] - - yearmonths = cutout.coords["year-month"].to_index() - - if isinstance(show_progress, string_types): - prefix = show_progress - else: - func_name = ( - convert_func.__name__[len("convert_") :] - if convert_func.__name__.startswith("convert_") - else convert_func.__name__ - ) - prefix = f"Convert `{func_name}`: " - - pbar = tqdm if show_progress else lambda x, desc: x - for ym in pbar(yearmonths, desc=prefix): - with xr.open_dataset(cutout.datasetfn(ym)) as ds: - if "view" in cutout.meta.attrs: - if isinstance(cutout.meta.attrs["view"], str): - cutout.meta.attrs["view"] = {} - cutout.meta.attrs.setdefault("view", {})["x"] = slice( - min(cutout.meta.coords["x"]).values.tolist(), - max(cutout.meta.coords["x"]).values.tolist(), - ) - cutout.meta.attrs.setdefault("view", {})["y"] = slice( - min(cutout.meta.coords["y"]).values.tolist(), - max(cutout.meta.coords["y"]).values.tolist(), - ) - ds = ds.sel(**cutout.meta.attrs["view"]) - - da = convert_func(ds, **convert_kwds) - results.append(da.load()) - - results = xr.concat(results, dim="time") - - return results - - -## temperature - - -def convert_temperature(ds): - """Return outside temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - """ - - # Temperature is in Kelvin - return ds["temperature"] - 273.15 - - -def temperature(cutout, **params): - return cutout.convert_cutout(convert_func=convert_temperature, **params) - - -## soil temperature - +if TYPE_CHECKING: + from .cutout import Cutout +else: + Cutout = object -def convert_soil_temperature(ds): - """Return soil temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - """ - - # Temperature is in Kelvin - - # There are nans where there is sea; by setting them - # to zero we guarantee they do not contribute when multiplied - # by matrix in geodata/aggregate.py - return (ds["soil temperature"] - 273.15).fillna(0.0) - - -def soil_temperature(cutout, **params): - return cutout.convert_cutout(convert_func=convert_soil_temperature, **params) - - -## heat demand +logger = logging.getLogger(__name__) -def convert_heat_demand(ds, threshold, a, constant, hour_shift): +# Heat Demand +def convert_heat_demand( + ds: xr.Dataset, + threshold: float, + a: float, + constant: float, + hour_shift: float, +): # Temperature is in Kelvin; take daily average T = ds["temperature"] - T.coords["time"].values += np.timedelta64(dt.timedelta(hours=hour_shift)) + T.coords["time"] += np.timedelta64(dt.timedelta(hours=hour_shift)) T = ds["temperature"].resample(time="1D").mean(dim="time") threshold += 273.15 @@ -157,62 +67,6 @@ def convert_heat_demand(ds, threshold, a, constant, hour_shift): return constant + heat_demand_value -def heat_demand(cutout, threshold=15.0, a=1.0, constant=0.0, hour_shift=0.0, **params): - """ - Convert outside temperature into daily heat demand using the - degree-day approximation. - - Since "daily average temperature" means different things in - different time zones and since xarray coordinates do not handle - time zones gracefully like pd.DateTimeIndex, you can provide an - hour_shift to redefine when the day starts. - - E.g. for Moscow in winter, hour_shift = 4, for New York in winter, - hour_shift = -5 - - This time shift applies across the entire spatial scope of ds for - all times. More fine-grained control will be built in a some - point, i.e. space- and time-dependent time zones. - - WARNING: Because the original data is provided every month, at the - month boundaries there is untidiness if you use a time shift. The - resulting xarray will have duplicates in the index for the parts - of the day in each month at the boundary. You will have to - re-average these based on the number of hours in each month for - the duplicated day. - - Parameters - ---------- - threshold : float - Outside temperature in degrees Celsius above which there is no - heat demand. - a : float - Linear factor relating heat demand to outside temperature. - constant : float - Constant part of heat demand that does not depend on outside - temperature (e.g. due to water heating). - hour_shift : float - Time shift relative to UTC for taking daily average - - Note - ---- - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - """ - - return cutout.convert_cutout( - convert_func=convert_heat_demand, - threshold=threshold, - a=a, - constant=constant, - hour_shift=hour_shift, - **params, - ) - - -## solar thermal collectors - - def convert_solar_thermal( ds, orientation, trigon_model, clearsky_model, c0, c1, t_store ): @@ -232,7 +86,7 @@ def convert_solar_thermal( output = irradiation * eta - return (output).where(output > 0.0).fillna(0.0) + return output.where(output > 0.0).fillna(0.0) def convert_pv(ds, panel, orientation, trigon_model="simple", clearsky_model="simple"): @@ -249,9 +103,6 @@ def convert_pv(ds, panel, orientation, trigon_model="simple", clearsky_model="si return solar_panel -## wind - - def convert_wind(ds, turbine, **params): """ Convert wind speeds for turbine to wind energy generation. @@ -271,7 +122,6 @@ def convert_wind(ds, turbine, **params): """ V, POW, hub_height, P = itemgetter("V", "POW", "hub_height", "P")(turbine) - wnd_hub = windm.extrapolate_wind_speed(ds, to_height=hub_height, **params) return xr.DataArray(np.interp(wnd_hub, V, POW / P), coords=wnd_hub.coords) @@ -357,50 +207,334 @@ def convert_pm25(ds): return 1e9 * ds["pm25"] # kg / m3 to ug / m3 -# Manipulate arbitrary variables +def heat_demand( + cutout: Cutout, + threshold: float = 15.0, + a: float = 1.0, + constant: float = 0.0, + hour_shift: float = 0.0, + **params, +): + """Convert outside temperature into daily heat demand using the + degree-day approximation. + + Since "daily average temperature" means different things in + different time zones and since xarray coordinates do not handle + time zones gracefully like pd.DateTimeIndex, you can provide an + hour_shift to redefine when the day starts. + E.g. for Moscow in winter, hour_shift = 4, for New York in winter, + hour_shift = -5 -def _get_var(ds, var): + This time shift applies across the entire spatial scope of ds for + all times. More fine-grained control will be built in a some + point, i.e. space- and time-dependent time zones. + + WARNING: Because the original data is provided every month, at the + month boundaries there is untidiness if you use a time shift. The + resulting xarray will have duplicates in the index for the parts + of the day in each month at the boundary. You will have to + re-average these based on the number of hours in each month for + the duplicated day. + + Args: + threshold (float): Outside temperature in degrees Celsius above which there is no heat demand. + a (float): Linear factor relating heat demand to outside temperature. + constant (float): Constant part of heat demand that does not depend on outside + temperature (e.g. due to water heating). + hour_shift (float): Time shift relative to UTC for taking daily average + + Returns: + xr.DataArray: Heat demand + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. """ - (Internal) Extract a specific variable from cutout - See: get_var + + return cutout._convert_cutout( + convert_func=convert_heat_demand, + threshold=threshold, + a=a, + constant=constant, + hour_shift=hour_shift, + **params, + ) + + +def temperature(cutout: Cutout, **convert_params): + """Convert temperature in Cutout to outside temperature. + + Args: + convert_params: Keyword arguments passed to `convert_cutout` function + + Returns: + xr.DataArray: Data of the Cutout with temperature converted to outside temperatures. """ - return xr.DataArray(ds[var], coords=ds.coords) + return cutout._convert_cutout( + convert_func=lambda ds: ds["temperature"] - 273.15, **convert_params + ) -def get_var(cutout, var, **params): +def soil_temperature(cutout: Cutout, **convert_params): + """Return soil temperature (useful for e.g. heat pump T-dependent + coefficient of performance). + + Args: + convert_params: Keyword arguments passed to `convert_cutout` function + + Returns: + xr.DataArray: Data of the Cutout with temperature converted to soil temperatures. """ - Extract a specific variable from cutout + return cutout._convert_cutout( + convert_func=lambda ds: (ds["soil temperature"] - 273.15).fillna(0.0), + **convert_params, + ) - Parameters - ---------- - var : str - Name of variable to extract from dataset - Returns: dataarray +def solar_thermal( + cutout: Cutout, + orientation: dict | str | Callable | None = None, + trigon_model: str = "simple", + clearsky_model: Literal["simple", "enhanced"] = "simple", + c0: float = 0.8, + c1: float = 3.0, + t_store: float = 80.0, + **params, +): + """Convert downward short-wave radiation flux and outside temperature + into time series for solar thermal collectors. + + Mathematical model and defaults for c0, c1 based on model in [1]. + + Args: + orientation (Union[dict, str, callable]): Panel orientation with slope and azimuth + (units of degrees), or 'latitude_optimal'. + trigon_model (str): Type of trigonometry model + clearsky_model (str): Type of clearsky model for diffuse irradiation. Either + `simple` or `enhanced`. + c0 (float): Parameter for model in [1] This defaults to 0.8. + c1 (float): Parameter for model in [1] This defaults to 3.0. + t_store (float): Store temperature in degree Celsius + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + + References: + [1] Henning and Palzer, Renewable and Sustainable Energy Reviews 30 + (2014) 1003-1018 """ - logger.info("Getting variable: %s", str(var)) - return cutout._convert_cutout(convert_func=_get_var, var=var, **params) + + if orientation is None: + orientation = {"slope": 45.0, "azimuth": 180.0} + + if not callable(orientation): + orientation = get_orientation(orientation) + + return cutout._convert_cutout( + convert_func=convert_solar_thermal, + orientation=orientation, + trigon_model=trigon_model, + clearsky_model=clearsky_model, + c0=c0, + c1=c1, + t_store=t_store, + **params, + ) -def _compute_var(ds, fn): +def wind( + cutout: Cutout, + turbine: str | dict, + method: Literal["simple", "interpolation", "extrapolation"], + smooth: bool | dict = False, + **params, +): + """Convert wind speed time-series into wind generation time-series. + + Args: + turbine (Union[str, dict]): Name of a turbine or a dictionary with the parameters + for the wind turbine in [2]. + smooth (Union[bool, dict]): If True, the wind speed time-series will be smoothed + before conversion. If False, no smoothing will be applied. If a dictionary is + passed, the smoothing parameters will be used. + **params: Keyword arguments passed to `convert_cutout` function """ - (Internal) Compute a specific function from cutout - See: compute_var + + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + + if smooth: + turbine = windturbine_smooth(turbine, params=smooth) + + match method: + case "simple": + return cutout._convert_cutout( + convert_func=convert_wind, turbine=turbine, **params + ) + + case _: + raise ValueError(f"Method {method} not supported.") + + +def windspd(cutout: Cutout, **params): """ - return xr.DataArray(fn(ds), coords=ds.coords) + Generate wind speed time-series + convert.convert_cutout → convert.convert_windspd + + Parameters + ---------- + **params + Must have 1 of: + turbine : str or dict + Name of a turbine + hub_height : num + Extrapolation height + + Can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + e.g. var_height='lml' -def compute_var(cutout, fn, **params): """ - Compute a specific function from cutout + + if "turbine" in params: + turbine = params.pop("turbine") + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + else: + raise ValueError(f"Turbine ({turbine}) not found.") + hub_height = itemgetter("hub_height")(turbine) + elif "hub_height" in params: + hub_height = params.pop("hub_height") + elif "to_height" in params: + hub_height = params.pop("to_height") + else: + raise ValueError("Either a turbine or hub_height must be specified.") + + params["hub_height"] = hub_height + + return cutout._convert_cutout(convert_func=convert_windspd, **params) + + +def windwpd(cutout: Cutout, **params): + """ + Generate wind power density time-series + + convert.convert_cutout → convert.convert_windwpd Parameters ---------- - var : str - Name of variable to extract from dataset + **params + Must have 1 of: + turbine : str or dict + Name of a turbine + hub_height : num + Extrapolation height + + Can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + e.g. var_height='lml' - Returns: dataarray """ - logger.info("Computing variable: %s", str(fn)) - return cutout.convert_cutout(convert_func=_compute_var, fn=fn, **params) + + if "turbine" in params: + turbine = params.pop("turbine") + if isinstance(turbine, str): + turbine = get_windturbineconfig(turbine) + else: + raise ValueError(f"Turbine ({turbine}) not found.") + hub_height = itemgetter("hub_height")(turbine) + elif "hub_height" in params: + hub_height = params.pop("hub_height") + elif "to_height" in params: + hub_height = params.pop("to_height") + else: + raise ValueError("Either a turbine or hub_height must be specified.") + + params["hub_height"] = hub_height + + return cutout._convert_cutout(convert_func=convert_windwpd, **params) + + +def pv( + cutout: Cutout, + panel: str | dict, + orientation: str | dict | Callable, + clearsky_model: str | None = None, + **params, +): + """Convert downward-shortwave, upward-shortwave radiation flux and + ambient temperature into a pv generation time-series. + + Args: + panel (Union[str, dict]): Panel name known to the reatlas client or a panel config + dictionary with the parameters for the electrical model in [3]. + orientation (Union[str, dict, callback]): Panel orientation can be chosen from either + 'latitude_optimal', a constant orientation {'slope': 0.0, + 'azimuth': 0.0} or a callback function with the same signature + as the callbacks generated by the + `geodata.pv.orientation.make_*` functions. + clearsky_model (Optional[str]): Either the 'simple' or the 'enhanced' Reindl clearsky + model. The default choice of None will choose dependending on + data availability, since the 'enhanced' model also + incorporates ambient air temperature and relative humidity. + + Returns: + xr.DataArray: Time-series or capacity factors based on additional general + conversion arguments. + + Note: + You can also specify all of the general conversion arguments + documented in the `convert_cutout` function. + + References: + [1] Soteris A. Kalogirou. Solar Energy Engineering: Processes and Systems, + pages 49-117,469-516. Academic Press, 2009. ISBN 0123745012. + [2] D.T. Reindl, W.A. Beckman, and J.A. Duffie. Diffuse fraction correla- + tions. Solar Energy, 45(1):1 - 7, 1990. + [3] Hans Georg Beyer, Gerd Heilscher and Stefan Bofinger. A Robust Model + for the MPP Performance of Different Types of PV-Modules Applied for + the Performance Check of Grid Connected Systems, Freiburg, June 2004. + Eurosun (ISES Europe Solar Congress). + """ + + if isinstance(panel, str): + panel = get_solarpanelconfig(panel) + if not callable(orientation): + orientation = get_orientation(orientation) + + return cutout._convert_cutout( + convert_func=convert_pv, + panel=panel, + orientation=orientation, + clearsky_model=clearsky_model, + **params, + ) + + +def pm25(cutout: Cutout, **params): + """ + Generate PM2.5 time series [ug / m3] + (see convert_pm25 for details) + + Returns: + xr.DataArray: PM2.5 time series + + """ + + return cutout._convert_cutout(convert_func=convert_pm25, **params) + + +__all__ = [ + "heat_demand", + "temperature", + "soil_temperature", + "solar_thermal", + "wind", + "windspd", + "windwpd", + "pv", + "pm25", +] diff --git a/src/geodata/cutout.py b/src/geodata/cutout.py index 058f2c33..507cf932 100644 --- a/src/geodata/cutout.py +++ b/src/geodata/cutout.py @@ -1,6 +1,6 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) # Copyright 2020 Michael Davidson (UCSD), William Honaker, Jiahe Feng (UCSD), Yuanbo Shi -# Copyright 2023-2024 Xiqiang Liu +# Copyright 2023-2025 Xiqiang Liu # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -19,13 +19,8 @@ Cutout class to handle a subset of a Dataset. """ -import datetime as dt import logging -import os -import sys -from collections.abc import Iterable from functools import partial -from operator import itemgetter from pathlib import Path from typing import Literal, Optional, Union @@ -36,23 +31,19 @@ from shapely.geometry import box from tqdm.auto import tqdm - from . import config from .convert import ( - convert_pm25, - convert_pv, - convert_solar_thermal, - convert_wind, - convert_windspd, - convert_windwpd, - get_orientation, -) - -from .resource import ( - get_solarpanelconfig, - get_windturbineconfig, - windturbine_smooth, + heat_demand, + pm25, + pv, + soil_temperature, + solar_thermal, + temperature, + wind, + windspd, + windwpd, ) +from .datasets._base import BaseDataset from .mask import Mask from .preparation import ( cutout_get_meta, @@ -60,6 +51,8 @@ cutout_prepare, cutout_produce_specific_dataseries, ) +from .types import BoundRange, DateRange +from .utils import ensure_slice logger = logging.getLogger(__name__) @@ -68,13 +61,15 @@ class Cutout: """Cutout class to handle a subset of a Dataset. Args: - module (Literal["era5", "merra2"]): name of the dataset module to use. - weather_data_config (str): name of the weather data config to use. - the name will be automatically generated. + name (str): Name of the cutout. This name will be used to uniquely + identify the cutout. If a cutout with the same name already exists, + the existing cutout will be retrieved. + dataset_cls (type[BaseDataset]): Dataset class to use for the cutout. years (slice): years of the cutout. - name (Optional[str]): name of the cutout. Optional. If not specified, - cutout_dir (str): path to the cutout directory. Defaults to config.cutout_dir. - bounds (Optional[Iterable]): bounds of the cutout. Optional. If not specified, + cutout_dir (str): path to the cutout directory. This is optional. If not + specified, the cutout will be stored in the default cutout directory + under `GEODATA_ROOT`. + bounds (Optional[Sequence]): bounds of the cutout. Optional. If not specified, the bounds will be automatically generated. months (Optional[slice]): months of the cutout. Optional. If not specified, the months will be automatically generated. @@ -86,37 +81,39 @@ class Cutout: def __init__( self, - module: Literal["era5", "merra2"], - weather_data_config: str, - years: slice, - name: Optional[str] = None, + name: str, + dataset_cls: type[BaseDataset], + years: DateRange, cutout_dir: Union[str, Path] = config.cutout_dir, - bounds: Optional[Iterable] = None, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, + bounds: Optional[BoundRange] = None, + months: Optional[DateRange] = None, + xs: Optional[BoundRange] = None, + ys: Optional[BoundRange] = None, ): self.name = name - self.cutout_dir = os.path.join(cutout_dir, name) + self.cutout_dir = Path(cutout_dir, name) + + self.dataset_cls = dataset_cls + self.prepared = False self.empty = False - self.meta_append = 0 - self.config = weather_data_config self.meta = None self.merged_mask = None self.shape_mask = None self.area = None params_dict = { - "module": module, - "years": years, + "years": ensure_slice(years), "months": months, "xs": xs, "ys": ys, } + if bounds is None and (xs is None and ys is None): + raise TypeError("Either bounds or xs/ys arguments must be specified.") if bounds is not None and (xs is not None or ys is not None): raise TypeError("Cannot specify both bounds and xs/ys arguments.") + if bounds is not None: # if passed bounds array instead of xs, ys slices x1, y1, x2, y2 = bounds @@ -125,40 +122,15 @@ def __init__( if months is None: logger.info("No months specified, defaulting to 1-12") params_dict.update(months=slice(1, 12)) + params_dict["months"] = ensure_slice(months) - if os.path.isdir(self.cutout_dir): - # If cutout dir exists, check completness of files - if os.path.isfile(self.datasetfn()): # open existing meta file - self.meta = xr.open_dataset(self.datasetfn()).stack( - **{"year-month": ("year", "month")} - ) - - if ( - self.meta is not None - and "years" in params_dict - and "months" in params_dict - and all( - os.path.isfile(self.datasetfn([y, m])) - for y in range( - params_dict["years"].start, params_dict["years"].stop + 1 - ) - for m in range( - params_dict["months"].start, params_dict["months"].stop + 1 - ) - ) - ): - # All files are accounted for. Checking basic data and coverage - if "module" not in self.meta.attrs: - raise TypeError("No module given in meta file of cutout.") - - # load dataset module based on file metadata - from geodata import Dataset - - self.dataset_module: Dataset = sys.modules[ - "geodata.datasets." + self.meta.attrs["module"] - ] - params_dict["module"] = self.meta.attrs["module"] + self.prepared = False + if self.cutout_dir.is_dir() and self.meta_path.is_file(): + self.meta = xr.open_dataset(self._get_filename()).stack( + dim={"year-month": ("year", "month")} + ) + if all(f.is_file() for f in self.catalog): logger.info("All cutout (%s, %s) files available.", name, cutout_dir) # At least one of xs, ys is in params_dict @@ -169,130 +141,103 @@ def __init__( # Subset is available self.prepared = True logger.info("Cutout subset prepared: %s", self) + return else: logger.info("Cutout subset not available: %s", self) else: # No subsetting of bounds. Keep full cutout self.prepared = True logger.info("Cutout prepared: %s", self) + return + + logger.info("Cutout (%s, %s) not complete.", name, cutout_dir) + if {"xs", "ys", "years"}.difference(params_dict): + raise TypeError( + "Arguments `xs`, `ys`, and `years` need to be specified to create " + "a cutout." + ) - else: - # Not all files accounted for - self.prepared = False - logger.info("Cutout (%s, %s) not complete.", name, cutout_dir) + if self.meta is not None: + # if meta.nc exists, close and delete it + self.meta.close() + self.meta_path.unlink() - if not self.prepared: - # Still need to prepare cutout - if "module" not in params_dict: - raise TypeError("Module is required to create cutout.") - # load module from geodata library - self.dataset_module = sys.modules[ - "geodata.datasets." + params_dict["module"] - ] - - logger.info("Cutout (%s, %s) not found or incomplete.", name, cutout_dir) - - if {"xs", "ys", "years"}.difference(params_dict): - raise TypeError( - "Arguments `xs`, `ys`, and `years` need to be specified for a cutout." - ) - - if self.meta is not None: - # if meta.nc exists, close and delete it - self.meta.close() - os.remove(self.datasetfn()) - - ## Main preparation call for metadata - # preparation.cutout_get_meta - # cutout.meta_data_config - # dataset_module.meta_data_config (e.g. prepare_meta_era5) - self.meta = self.get_meta(**params_dict) - - # Ensure cutout directory exists - if not os.path.isdir(self.cutout_dir): - os.mkdir(self.cutout_dir) - - # Write meta file - self.meta_clean.unstack("year-month").to_netcdf(self.datasetfn()) - - def datasetfn(self, *args): + self.meta = self.get_meta(**params_dict) + self.cutout_dir.mkdir(parents=True, exist_ok=True) + + # Write meta file + self.meta_clean.unstack("year-month").to_netcdf(self.meta_path) + + def _get_filename( + self, year: int | tuple[int, int] | None = None, month: int | None = None + ): """Return path to dataset xarray files related to this Cutout. + If both year and month are None, return path to meta.nc file. Args: - *args: optional arguments to append to the filename. If not specified, - the meta file will be returned. If specified, the dataset file will be returned, depending - on the number of arguments. One argument will return the dataset file for the given - year-month string, two arguments will return the dataset file for the given year and month. + year (int): The year to get the filename for. Defaults to None. + month (int): The month to get the filename for. Defaults to None. Returns: - str: path to dataset xarray files related to this Cutout. + Path: Path to the dataset xarray file. """ - dataset = None - if len(args) == 2: - dataset = args - elif len(args) == 1: - dataset = args[0] - else: - dataset = None - return os.path.join( - # pylint: disable=consider-using-f-string - self.cutout_dir, - ("meta.nc" if dataset is None else "{}{:0>2}.nc".format(*dataset)), - # pylint: enable=consider-using-f-string - ) + if year is None and month is None: + return self.cutout_dir / "meta.nc" + + if year is not None: + if month is None: + year, month = year + + return self.cutout_dir / f"{year}{month:0>2}.nc" + + @property + def catalog(self): + """A generator that yields all dataset files.""" + + for year in self.coords["year"].values: + for month in self.coords["month"].values: + yield self._get_filename(year, month) + + @property + def meta_path(self): + """Path to the metadata file.""" + + return self._get_filename() @property def meta_data_config(self): """Metadata configuration for the Cutout""" - return dict( - tasks_func=self.dataset_module.weather_data_config[self.config][ - "tasks_func" - ], - prepare_func=self.dataset_module.weather_data_config[self.config][ - "meta_prepare_func" - ], - template=self.dataset_module.weather_data_config[self.config]["template"], - file_granularity=self.dataset_module.weather_data_config[self.config][ - "file_granularity" - ], - ) + return { + "tasks_func": self.dataset_cls.tasks_func, + "prepare_func": self.dataset_cls.meta_prepare_func, + "file_granularity": self.dataset_cls.frequency, + } @property - def weather_data_config(self): + def weather_config(self): """The weather data configuration for the Cutout.""" - return self.dataset_module.weather_data_config - @property - def variables(self): - """The variables contained in the Cutout.""" - return self.dataset_module.weather_data_config[self.config]["variables"] + return self.dataset_cls.weather_config @property def info(self): """Summary information about the Cutout.""" - return dict( - name=self.name, - config=self.config, - prepared=self.prepared, - projection=self.dataset_module.projection, - shape=[len(self.coords["y"]), len(self.coords["x"])], - extent=( - list(self.coords["x"].values[[0, -1]]) - + list(self.coords["y"].values[[-1, 0]]) - ), - dimensions=self.meta.dims, - coordinates=self.meta.coords, - variables=self.dataset_module.weather_data_config[self.config]["variables"], - dataset_module=self.dataset_module, - cutout_dir=self.cutout_dir, - ) + return { + "name": self.name, + "prepared": self.prepared, + "shape": self.shape, + "extent": self.extent, + "years": self.years, + "months": self.months, + "meta": self.meta, + } @property def projection(self): """The projection of the Cutout.""" - return self.dataset_module.projection + return self.dataset_cls.projection @property def coords(self): @@ -305,7 +250,7 @@ def meta_clean(self): meta = self.meta if meta.attrs.get("view", {}): view = {} - for name, value in meta.attrs.get("view", {}).items(): + for name, value in meta.attrs["view"].items(): view.update({name: [value.start, value.stop]}) meta.attrs["view"] = str(view) return meta @@ -345,7 +290,6 @@ def grid_cells(self): def __repr__(self): yearmonths = self.coords["year-month"].to_index() - # pylint: disable=consider-using-f-string return "".format( self.name, self.coords["x"].values[0], @@ -358,7 +302,6 @@ def __repr__(self): yearmonths[-1][1], "" if self.prepared else "UN", ) - # pylint: enable=consider-using-f-string def add_mask(self, name: str, merged_mask: bool = True, shape_mask: bool = True): """Add mask attribute to the cutout, from a previously saved mask objects. @@ -500,7 +443,7 @@ def mask( return res # Preparation functions - get_meta = cutout_get_meta # preparation.cutout_get_meta + get_meta = cutout_get_meta get_meta_view = cutout_get_meta_view # preparation.cutout_get_meta_view prepare = cutout_prepare # preparation.cutout_prepare produce_specific_dataseries = cutout_produce_specific_dataseries @@ -539,12 +482,12 @@ def _convert_cutout( if convert_func.__name__.startswith("convert_") else convert_func.__name__ ) - prefix = f"Convert `{func_name}`: " + prefix = f"Convert {func_name}" for ym in tqdm( yearmonths, desc=prefix, disable=not show_progress, dynamic_ncols=True ): - with xr.open_dataset(self.datasetfn(ym)) as ds: + with xr.open_dataset(self._get_filename(ym)) as ds: if "view" in self.meta.attrs: if isinstance(self.meta.attrs["view"], str): self.meta.attrs["view"] = {} @@ -563,346 +506,15 @@ def _convert_cutout( return xr.concat(results, dim="time") - def heat_demand( - self, - threshold: float = 15.0, - a: float = 1.0, - constant: float = 0.0, - hour_shift: float = 0.0, - **params, - ): - """Convert outside temperature into daily heat demand using the - degree-day approximation. - - Since "daily average temperature" means different things in - different time zones and since xarray coordinates do not handle - time zones gracefully like pd.DateTimeIndex, you can provide an - hour_shift to redefine when the day starts. - - E.g. for Moscow in winter, hour_shift = 4, for New York in winter, - hour_shift = -5 - - This time shift applies across the entire spatial scope of ds for - all times. More fine-grained control will be built in a some - point, i.e. space- and time-dependent time zones. - - WARNING: Because the original data is provided every month, at the - month boundaries there is untidiness if you use a time shift. The - resulting xarray will have duplicates in the index for the parts - of the day in each month at the boundary. You will have to - re-average these based on the number of hours in each month for - the duplicated day. - - Args: - threshold (float): Outside temperature in degrees Celsius above which there is no heat demand. - a (float): Linear factor relating heat demand to outside temperature. - constant (float): Constant part of heat demand that does not depend on outside - temperature (e.g. due to water heating). - hour_shift (float): Time shift relative to UTC for taking daily average - - Returns: - xr.DataArray: Heat demand - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - """ - - def convert_heat_demand( - ds: xr.Dataset, - threshold: float, - a: float, - constant: float, - hour_shift: float, - ): - # Temperature is in Kelvin; take daily average - T = ds["temperature"] - T.coords["time"].values += np.timedelta64(dt.timedelta(hours=hour_shift)) - - T = ds["temperature"].resample(time="1D").mean(dim="time") - threshold += 273.15 - heat_demand_value = a * (threshold - T) - - heat_demand_value.values[heat_demand_value.values < 0.0] = 0.0 - - return constant + heat_demand_value - - return self._convert_cutout( - convert_func=convert_heat_demand, - threshold=threshold, - a=a, - constant=constant, - hour_shift=hour_shift, - **params, - ) - - def temperature(self, **convert_params): - """Convert temperature in Cutout to outside temperature. - - Args: - convert_params: Keyword arguments passed to `convert_cutout` function - - Returns: - xr.DataArray: Data of the Cutout with temperature converted to outside temperatures. - """ - return self._convert_cutout( - convert_func=lambda ds: ds["temperature"] - 273.15, **convert_params - ) - - def soil_temperature(self, **convert_params): - """Return soil temperature (useful for e.g. heat pump T-dependent - coefficient of performance). - - Args: - convert_params: Keyword arguments passed to `convert_cutout` function - - Returns: - xr.DataArray: Data of the Cutout with temperature converted to soil temperatures. - """ - return self._convert_cutout( - convert_func=lambda ds: (ds["soil temperature"] - 273.15).fillna(0.0), - **convert_params, - ) - - def solar_thermal( - self, - orientation: Optional[Union[dict, str, callable]] = None, - trigon_model: str = "simple", - clearsky_model: Literal["simple", "enhanced"] = "simple", - c0: float = 0.8, - c1: float = 3.0, - t_store: float = 80.0, - **params, - ): - """Convert downward short-wave radiation flux and outside temperature - into time series for solar thermal collectors. - - Mathematical model and defaults for c0, c1 based on model in [1]. - - Args: - orientation (Union[dict, str, callable]): Panel orientation with slope and azimuth - (units of degrees), or 'latitude_optimal'. - trigon_model (str): Type of trigonometry model - clearsky_model (str): Type of clearsky model for diffuse irradiation. Either - `simple` or `enhanced`. - c0 (float): Parameter for model in [1] This defaults to 0.8. - c1 (float): Parameter for model in [1] This defaults to 3.0. - t_store (float): Store temperature in degree Celsius - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Henning and Palzer, Renewable and Sustainable Energy Reviews 30 - (2014) 1003-1018 - """ - - if orientation is None: - orientation = {"slope": 45.0, "azimuth": 180.0} - - if not callable(orientation): - orientation = get_orientation(orientation) - - return self._convert_cutout( - convert_func=convert_solar_thermal, - orientation=orientation, - trigon_model=trigon_model, - clearsky_model=clearsky_model, - c0=c0, - c1=c1, - t_store=t_store, - **params, - ) - - # NOTE: The following wind-related functions will be deprecated in the future - # in favor of the wind modeling module. - def wind( - self, turbine: Union[str, dict], smooth: Union[bool, dict] = False, **params - ): - """ - Generate wind generation time-series - - - loads turbine dict based on passed parameters (resource.get_windturbineconfig) - - optionally, smooths turbine power curve (resource.windturbine_smooth) - - calls convert_wind (convert.convert_cutout) - - Args: - turbine (Union[str, dict]): Name of a turbine known by the reatlas client or a - turbineconfig dictionary with the keys 'hub_height' for the - hub height and 'V', 'POW' defining the power curve. - smooth (Union[bool, dict]): If True smooth power curve with a gaussian kernel as - determined for the Danish wind fleet to Delta_v = 1.27 and - sigma = 2.s29. A dict allows to tune these values. - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Andresen G B, Søndergaard A A and Greiner M 2015 Energy 93, Part 1 - 1074 - 1088. doi:10.1016/j.energy.2015.09.071 - """ - - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - - if smooth: - turbine = windturbine_smooth(turbine, params=smooth) - - return self._convert_cutout( - convert_func=convert_wind, turbine=turbine, **params - ) - - def windspd(self, **params): - """ - Generate wind speed time-series - - convert.convert_cutout → convert.convert_windspd - - Parameters - ---------- - **params - Must have 1 of: - turbine : str or dict - Name of a turbine - hub_height : num - Extrapolation height - - Can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - e.g. var_height='lml' - - """ - - if "turbine" in params: - turbine = params.pop("turbine") - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - else: - raise ValueError(f"Turbine ({turbine}) not found.") - hub_height = itemgetter("hub_height")(turbine) - elif "hub_height" in params: - hub_height = params.pop("hub_height") - elif "to_height" in params: - hub_height = params.pop("to_height") - else: - raise ValueError("Either a turbine or hub_height must be specified.") - - params["hub_height"] = hub_height - - return self._convert_cutout(convert_func=convert_windspd, **params) - - def windwpd(self, **params): - """ - Generate wind power density time-series - - convert.convert_cutout → convert.convert_windwpd - - Parameters - ---------- - **params - Must have 1 of: - turbine : str or dict - Name of a turbine - hub_height : num - Extrapolation height - - Can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - e.g. var_height='lml' - - """ - - if "turbine" in params: - turbine = params.pop("turbine") - if isinstance(turbine, str): - turbine = get_windturbineconfig(turbine) - else: - raise ValueError(f"Turbine ({turbine}) not found.") - hub_height = itemgetter("hub_height")(turbine) - elif "hub_height" in params: - hub_height = params.pop("hub_height") - elif "to_height" in params: - hub_height = params.pop("to_height") - else: - raise ValueError("Either a turbine or hub_height must be specified.") - - params["hub_height"] = hub_height - - return self._convert_cutout(convert_func=convert_windwpd, **params) - - def pv( - self, - panel: Union[str, dict], - orientation: Union[str, dict, callable], - clearsky_model: Optional[str] = None, - **params, - ): - """Convert downward-shortwave, upward-shortwave radiation flux and - ambient temperature into a pv generation time-series. - - Args: - panel (Union[str, dict]): Panel name known to the reatlas client or a panel config - dictionary with the parameters for the electrical model in [3]. - orientation (Union[str, dict, callback]): Panel orientation can be chosen from either - 'latitude_optimal', a constant orientation {'slope': 0.0, - 'azimuth': 0.0} or a callback function with the same signature - as the callbacks generated by the - `geodata.pv.orientation.make_*` functions. - clearsky_model (Optional[str]): Either the 'simple' or the 'enhanced' Reindl clearsky - model. The default choice of None will choose dependending on - data availability, since the 'enhanced' model also - incorporates ambient air temperature and relative humidity. - - Returns: - xr.DataArray: Time-series or capacity factors based on additional general - conversion arguments. - - Note: - You can also specify all of the general conversion arguments - documented in the `convert_cutout` function. - - References: - [1] Soteris A. Kalogirou. Solar Energy Engineering: Processes and Systems, - pages 49-117,469-516. Academic Press, 2009. ISBN 0123745012. - [2] D.T. Reindl, W.A. Beckman, and J.A. Duffie. Diffuse fraction correla- - tions. Solar Energy, 45(1):1 - 7, 1990. - [3] Hans Georg Beyer, Gerd Heilscher and Stefan Bofinger. A Robust Model - for the MPP Performance of Different Types of PV-Modules Applied for - the Performance Check of Grid Connected Systems, Freiburg, June 2004. - Eurosun (ISES Europe Solar Congress). - """ - - if isinstance(panel, str): - panel = get_solarpanelconfig(panel) - if not callable(orientation): - orientation = get_orientation(orientation) - - return self._convert_cutout( - convert_func=convert_pv, - panel=panel, - orientation=orientation, - clearsky_model=clearsky_model, - **params, - ) - - def pm25(self, **params): - """ - Generate PM2.5 time series [ug / m3] - (see convert_pm25 for details) - - Parameters - ---------- - **params : None needed currently. - - Returns - ------- - pm25 : xr.DataArray - - """ - - return self._convert_cutout(convert_func=convert_pm25, **params) + heat_demand = heat_demand + temperature = temperature + soil_temperature = soil_temperature + solar_thermal = solar_thermal + wind = wind + windspd = windspd + windwpd = windwpd + pm25 = pm25 + pv = pv def ds_reformat_index(ds: xr.DataArray) -> xr.DataArray: @@ -946,7 +558,7 @@ def _find_intercept(list1, list2, start, threshold=0): if min_res == init: return 0 else: - return i # type: ignore + return i def coarsen(ori: xr.Dataset, tar: xr.Dataset, func: Literal["sum", "mean"] = "mean"): @@ -998,7 +610,7 @@ def coarsen(ori: xr.Dataset, tar: xr.Dataset, func: Literal["sum", "mean"] = "me else: raise ValueError("func can only be 'mean' or 'sum'") - return coarsen.reindex_like(tar, method="nearest") + return _coarsen.reindex_like(tar, method="nearest") def calc_grid_area(lis_lats_lons): @@ -1044,3 +656,6 @@ def calc_shp_area(shp, shp_projection="+proj=latlon"): shp, ) return temp_shape.area / 1000000 + + +__all__ = ["Cutout", "coarsen", "calc_grid_area", "calc_shp_area"] diff --git a/src/geodata/datasets/__init__.py b/src/geodata/datasets/__init__.py index f874c8ad..4d5955fe 100644 --- a/src/geodata/datasets/__init__.py +++ b/src/geodata/datasets/__init__.py @@ -14,6 +14,68 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +"""A interface to download and manage geospatial datasets. + +Attributes: + registry (dict): Registry of available datasets. You can retrieve relevant dataset classes using `registry.get()`. + DatasetType (type): A type to aid type hinting dataset classes. +""" + from . import era5, merra2 +from ._base import DatasetType +from ._base import _registry as registry + +__all__ = [ + "era5", + "merra2", + "registry", + "register_hrrr", + "DatasetType", + "load_dataset", + "list_datasets", +] + + +def load_dataset(weather_data_config: str) -> DatasetType: + """Load a dataset based on the provided weather data configuration. + This function retrieves the dataset class from the registry based on the provided configuration string. + It is very similar to the `get` method of the registry (e.g. `registry.get(weather_data_config)`), + but it is more explicit in its intent to load a dataset. + + Args: + weather_data_config (str): The configuration string for the dataset. + + Returns: + DatasetType: An instance of the dataset class corresponding to the configuration. + + Raises: + ValueError: If the provided configuration is not registered in the registry. + + """ + if weather_data_config not in registry: + raise ValueError(f"Dataset '{weather_data_config}' is not registered.") + + return registry[weather_data_config] + + +def list_datasets() -> list[str]: + """List all registered datasets. + + Returns: + list[str]: A list of dataset names that are registered in the registry. + + Raises: + ValueError: If no datasets are registered in the registry. + """ + + if not registry: + raise ValueError("No datasets are registered in the registry.") + + return list(registry.keys()) + -__all__ = ["era5", "merra2"] +def register_hrrr(): + """Register the HRRR dataset with the registry. By default, the HRRR dataset is not registered to avoid import overhead with the HRRR Herbie package. + This function can be called to register the HRRR dataset when needed.""" + from . import hrrr # noqa: F401 diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py new file mode 100644 index 00000000..8ac18bd2 --- /dev/null +++ b/src/geodata/datasets/_base.py @@ -0,0 +1,622 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import dataclasses +import hashlib +import itertools +import logging +from collections.abc import Sequence +from pathlib import Path +from typing import Literal, Type, TypeVar + +import pandas as pd +import xarray as xr +from tqdm.auto import tqdm + +from ..config import DATASET_ROOT_PATH +from ..types import BoundRange, CoordRange, DateRange + +logger = logging.getLogger(__name__) +DatasetType = TypeVar("DatasetType", bound="BaseDataset") + +_registry: dict[str, DatasetType] = {} + + +@dataclasses.dataclass +class AtomicDataset: + """AtomicDataset is a class that encapsulates an individual xarray file that was + downloaded. It provides a streamlined workflow for downloading, preprocessing, + and integrity checking of these datasets. + """ + + dataset: DatasetType + year: int + month: int + day: int | None = None + file_hash: str | None = None + url: str | tuple[str] | None = None + spinup: bool | None = None + + def __post_init__(self): + if not isinstance(self.dataset, BaseDataset): + raise ValueError("dataset must be an instance of BaseDataset") + + @property + def path(self): + """The path where the file should be saved""" + if self.day is None: + return self.dataset.storage_root / str(self.year) / f"{self.month:02d}.nc" + else: + return ( + self.dataset.storage_root + / str(self.year) + / f"{self.month:02d}" + / f"{self.day:02d}.nc" + ) + + def check(self, integrity: bool = False): + """Check the presence of the file and its integrity. + + Args: + integrity: A boolean flag indicating whether to check the integrity + of the file. If True, the file will be checked against its hash. + If False, only the presence of the file will be checked. + + Returns: + True if the file is present and its integrity is intact, False otherwise. + """ + + if not self.path.exists(): + logger.debug(f"{self.path} does not exist") + return False + + if not integrity: + return True + + # In case the file just got downloaded + if self.file_hash is None: + self.file_hash = self._compute_hash() + return True + + if self.file_hash != self._compute_hash(): + logger.warning(f"{self.path} is corrupted") + return False + + return True + + def _compute_hash(self): + """Compute the hash of the file. + + Returns: + The hash of the file. + """ + + hash_func = hashlib.sha256() + with open(self.path, "rb") as f: + while chunk := f.read(8192): # Read file in chunks + hash_func.update(chunk) + return hash_func.hexdigest() + + +class BaseDataset(abc.ABC): + """Dataset is a class that encapsulates any datasets natively supported + by geodata. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + + Args: + years: A slice object or sequence of two integers representing the + range of years to download. + months: A slice object or sequence of two integers representing the + range of months to download. + bounds (optional): A tuple of four floats representing the bounding box + (lon_min, lat_min, lon_max, lat_max) + **kwargs: Additional keyword arguments that are passed to the dataset. + + Notes: + - Subclasses of BaseDataset must define the following attributes: + - module: The module of the dataset. + - weather_config: The configuration of the dataset. + - Subclasses of BaseDataset must also implement the following methods: + - download: Method to download the dataset. + - _extra_setup: Method to handle any extra setup that is required + for the dataset. + - By default, the files downloaded by the dataset are defined to by monthly in + nature. That is, Subclasses can override this behavior by setting the `frequency` + + """ + + module: str + weather_config: str + frequency: Literal["hourly", "daily", "monthly"] = "monthly" + + def __init__( + self, + years: DateRange, + months: DateRange, + bounds: BoundRange | None = None, + **kwargs, + ): + if not hasattr(self, "module"): + raise ValueError("Subclasses of BaseDataset must define a module attribute") + + if not hasattr(self, "weather_config"): + raise ValueError( + "Subclasses of BaseDataset must define a weather_config attribute" + ) + + if not isinstance(years, slice): + if isinstance(years, Sequence): + if not all(isinstance(year, int) for year in years): + raise ValueError("years must be a sequence of integers") + elif not len(years) == 2: + raise ValueError("years must be a sequence of length 2") + years = slice(years[0], years[1]) + else: + raise ValueError( + f"""Invalid input {years} for years. Years must either be a + sequence of integers or a slice object""" + ) + self.years = years + + if not isinstance(months, slice): + if isinstance(months, Sequence): + if not all(isinstance(month, int) for month in months): + raise ValueError("months must be a sequence of integers") + elif not len(months) == 2: + raise ValueError("months must be a sequence of length 2") + elif not all(1 <= month <= 12 for month in months): + raise ValueError( + "months must be a sequence of integers between 1 and 12" + ) + months = slice(months[0], months[1]) + else: + raise ValueError( + f"""Invalid input {months} for months. Months must either be a + sequence of integers or a slice object""" + ) + self.months = months + + if bounds is not None: + if not all(isinstance(bound, (int, float)) for bound in bounds): + raise ValueError("bounds must be a sequence of integers or floats") + if not len(bounds) == 4: + raise ValueError("bounds must be a sequence of length 4") + if not all(-180 <= bound <= 180 for bound in [bounds[0], bounds[2]]): + raise ValueError("Longitude bounds must be between -180 and 180") + if not all(-90 <= bound <= 90 for bound in [bounds[1], bounds[3]]): + raise ValueError("Latitude bounds must be between -90 and 90") + self.bounds = bounds + + self.storage_root = DATASET_ROOT_PATH / self.module / self.weather_config + if not self.storage_root.exists(): + logger.info( + f"Storage directory for {self.__class__.__name__} does not exist, " + f"creating now at {self.storage_root}" + ) + self.storage_root.mkdir(parents=True) + + self._downloaded = False + + self._extra_kwargs = kwargs + self._extra_setup(**kwargs) + + def _extra_setup(self, **kwargs): + """Method to be implemented by subclasses to handle any extra setup + that is required for the dataset. + """ + + def _generate_manifest(self): + """Generate a manifest file for the dataset. This file contains + metadata about the dataset, including the file paths and their + integrity checks. + + TODO! This method is not ready yet! + + Returns: + A list of dictionaries containing the metadata of each file in the + dataset. + """ + + manifest = [] + for file in self.catalog: + manifest.append( + { + "path": str(file["save_path"]), + "integrity": file["downloaded"], + } + ) + + return manifest + + @property + def downloaded(self): + """A boolean flag indicating whether the dataset has been prepared + for use. This typically means that the dataset has been downloaded, + preprocessed, and stored in a format that is ready for use. + + The basic implementation of this method checks the presence of each file in + the catalog in the storage root. Subclasses can override this behavior if + a more comprehensive check is required. + """ + + if not self._downloaded: + self._downloaded = self._check_downloaded() + return self._downloaded + + def _check_downloaded(self): + # Check if the dataset is downloaded + for file in tqdm( + self.catalog, + unit="file", + dynamic_ncols=True, + desc="Checking Downloaded Files", + ): + if not file.check(): + logger.debug(f"{file.path} does not exist") + return False + + return True + + @abc.abstractmethod + def _download_file(self, file: AtomicDataset): + """Method to download a single file from the dataset. This method + should download the file and save it to the appropriate location. + + Args: + file: An instance of AtomicDataset representing the file to download. + """ + + def download(self, force: bool = False): + """Method to download the dataset. This method should download the + dataset files and store them in the appropriate location. If the dataset + is specified as a ``testing'' dataset, only the first three (3) days + of the dataset will be downloaded. + + Args: + force: A boolean flag indicating whether to force the download of + the dataset, even if it has already been downloaded. + """ + + if self.downloaded and not force: + logger.info(f"{self} has already been downloaded.") + return + + for file in tqdm(self.catalog, unit="file", dynamic_ncols=True): + # Skip the file if it has already been downloaded (unless force is True) + if file.check() and not force: + logger.debug(f"{file.path} already exists, skipping download") + continue + + # We first must ensure the directory exists + file.path.parent.mkdir(parents=True, exist_ok=True) + + self._download_file(file) + + if file.check(): + logger.debug("Postprocessing %s", file.path) + ds = xr.open_dataset(file.path).chunk("auto") + ds = self._rename_and_clean_coords(ds) + ds = self._dataset_postprocess(ds) + + # xarray does not support overwriting files, so we must save the + # dataset to a new file and then rename it backwards + ds.to_netcdf(file.path.with_stem(file.path.stem + "_postprocessed")) + ds.close() + + file.path.unlink() + file.path.with_stem(file.path.stem + "_postprocessed").rename(file.path) + + logger.info(f"Downloaded {self}") + logger.info("Cleaning and renaming coordinates") + + def _dataset_postprocess(self, ds: xr.Dataset | xr.DataArray, **kwargs): + """Method to postprocess the dataset after it has been downloaded. + This method should be implemented by subclasses to handle any + additional processing that is required for the dataset. + + Args: + ds: The dataset to postprocess. + **kwargs: Additional keyword arguments to pass to the function. + """ + + return ds + + def trim_variables( + self, + ds: xr.Dataset | xr.DataArray, + variables: Sequence[str] | None = None, + **kwargs, + ) -> xr.Dataset | xr.DataArray: + """Method to trim the dataset to only include the specified variables. + + Args: + ds: The dataset to trim. + variables: A sequence of strings representing the variables to keep. + If None, we will keep the variables specified in the `variables` + attribute of the dataset. + + Returns: + xr.Dataset | xr.DataArray: The trimmed dataset containing only the + specified variables. + + Raises: + ValueError: If the dataset does not have a `variables` attribute + defined and no variables are specified. + """ + + if variables is None: + if not hasattr(self, "variables"): + raise ValueError( + "The dataset does not have a `variables` attribute defined." + "Please specify the variables to keep." + ) + variables: Sequence[str] = getattr(self, "variables") + + return ds[variables] + + def __repr__(self): + return "".format( + self.module, + self.weather_config, + self.years.start, + self.years.stop, + self.months.start, + self.months.stop, + "Downloaded" if self.downloaded else "Not Downloaded", + " " + self.extra_repr if self.extra_repr else "", + ) + + @property + @abc.abstractmethod + def projection(self): + """The projection of the dataset. This should be a string that + represents the projection of the dataset. + """ + + @property + @abc.abstractmethod + def lat_direction(self) -> bool: + """Latitude direction stored in the dataset. This should be a boolean flag. + If True, the latitude increases from south to north. If False, the latitude + increases from north to south. + """ + + @property + def extra_repr(self): + return "" + + @property + def testing(self): + """A boolean flag indicating whether the dataset is being used for + testing. Under this mode, only the first few days or months of the dataset + will be downloaded (depending on the granularity). This is useful for + testing the dataset without downloading the entire dataset. + """ + + if "testing" not in self._extra_kwargs: + return False + if not isinstance(self._extra_kwargs["testing"], bool): + raise ValueError("testing must be a boolean flag") + return self._extra_kwargs["testing"] + + @property + def submodule(self): + """The submodule of the dataset. This can be defined by the dataset + using the `weather_config` attribute. If not defined, it will default + to the name of the dataset class. + """ + return getattr(self, "weather_config", self.__class__.__name__) + + @staticmethod + @abc.abstractmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + **meta_attrs, + ): + """A method that returns a list of tasks that can be run on the dataset.""" + + @staticmethod + @abc.abstractmethod + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + """A method that generates the metadata for the cutout.""" + + @staticmethod + @abc.abstractmethod + def prepare_func( + fn: str | Path, year: int, month: int, xs: CoordRange, ys: CoordRange, **kwargs + ): + """A method that prepares the cutout for individual dataset.""" + + @property + def catalog(self) -> list["AtomicDataset"]: + """A generator that yields all the files that need to be downloaded. + Each iteration should return a dictionary with the following keys: + - year: the year of the file + - month: the month of the file + - day: the day of the file (if applicable) + - hour: the hour of the file (if applicable) + - save_path: the path where the file should be saved + """ + + match self.frequency: + case "monthly": + cat = self._monthly_catalog() + case "daily": + cat = self._daily_catalog() + case _: + raise ValueError( + f"Invalid frequency {self.frequency} defined for this dataset." + ) + + return cat + + def get_monthly_catalog(self, year: int, month: int) -> list["AtomicDataset"]: + """Get the catalog for a specific month and year. + + Args: + year: The year of the file. + month: The month of the file. + """ + if not isinstance(year, int): + raise ValueError("year must be an integer") + if not isinstance(month, int): + raise ValueError("month must be an integer") + if not 1 <= month <= 12: + raise ValueError("month must be between 1 and 12") + + if not self.years.start <= year <= self.years.stop: + raise ValueError( + f"year must be between {self.years.start} and {self.years.stop}" + ) + + match self.frequency: + case "monthly": + return self._monthly_catalog(year, month) + case "daily": + return self._daily_catalog(year, month) + case _: + raise ValueError( + f"Invalid frequency {self.frequency} defined for this dataset." + ) + + def _monthly_catalog( + self, year: int | None = None, month: int | None = None + ) -> list["AtomicDataset"]: + if year is not None and month is not None: + return [AtomicDataset(self, year, month)] + if year is not None or month is not None: + raise ValueError( + "If one of year or month is specified, both must be specified." + ) + + catalog = [] + + for year, month in itertools.product( + range(self.years.start, self.years.stop + 1), + range(self.months.start, self.months.stop + 1), + ): + catalog.append(AtomicDataset(self, year, month)) + + return catalog + + def _daily_catalog( + self, year: int | None = None, month: int | None = None + ) -> list["AtomicDataset"]: + if year is not None and month is not None: + return [ + AtomicDataset(self, year, month, day) + for day in range( + 1, + pd.Timestamp(f"{year}-{month}-1").days_in_month + 1 + if not self.testing + else 3, + ) + ] + if year is not None or month is not None: + raise ValueError( + "If one of year or month is specified, both must be specified." + ) + + catalog = [] + + for year, month in itertools.product( + range(self.years.start, self.years.stop + 1), + range(self.months.start, self.months.stop + 1), + ): + for day in range( + 1, + pd.Timestamp(f"{year}-{month}-1").days_in_month + 1 + if not self.testing + else 3, + ): + catalog.append(AtomicDataset(self, year, month, day)) + + return catalog + + def _rename_and_clean_coords(self, ds: xr.Dataset, add_lon_lat: bool = False): + """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' + + Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. + + Args: + ds (xarray.Dataset): Dataset to rename + add_lon_lat (bool, optional): Add lon/lat columns. Defaults to False. + + Returns: + xarray.Dataset: Dataset with renamed coordinates + """ + + # Rename latitude / lat -> y, longitude / lon -> x + if "latitude" in ds.coords: + ds = ds.rename({"latitude": "y"}) + if "longitude" in ds.coords: + ds = ds.rename({"longitude": "x"}) + if "lat" in ds.coords: + ds = ds.rename({"lat": "y"}) + if "lon" in ds.coords: + ds = ds.rename({"lon": "x"}) + + # Flatten x and y if they are multi-dimensional + if ds.coords["x"].ndim > 1: + ds = ds.assign_coords(x=("x", ds.coords["x"][0].values)) + if ds.coords["y"].ndim > 1: + ds = ds.assign_coords(y=("y", ds.coords["y"][:, 0].values)) + + if add_lon_lat: + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + + return ds + + @classmethod + def _get_files(cls, year: int, month: int): + """Get the path where the file should be saved. Internal method used by Cutouts. + + Args: + year: The year of the file. + month: The month of the file. + """ + + storage_root = DATASET_ROOT_PATH / cls.module / cls.weather_config + + match cls.frequency: + case "monthly": + return [storage_root / str(year) / f"{month:02d}.nc"] + case "daily": + return list((storage_root / str(year) / f"{month:02d}").glob("*.nc")) + case _: + raise ValueError( + f"Invalid frequency {cls.frequency} defined for this dataset." + ) + + def __init_subclass__(cls: Type["BaseDataset"], **kwargs): + """Register the subclass in the registry. This allows us to + dynamically load the dataset from the module. + + Args: + cls: The class of the dataset. + """ + + super().__init_subclass__(**kwargs) + if not hasattr(cls, "weather_config"): + # Skip the class if it does not have a weather_config attribute + # This could be the case with intermediate base classes + return + + _registry[cls.weather_config] = cls + logger.debug(f"Registered {cls.weather_config} in the registry") diff --git a/src/geodata/datasets/era5.py b/src/geodata/datasets/era5.py deleted file mode 100644 index 6dd23b42..00000000 --- a/src/geodata/datasets/era5.py +++ /dev/null @@ -1,599 +0,0 @@ -# Copyright 2016-2017 Jonas Hoersch (FIAS), Tom Brown (FIAS), Markus Schlott (FIAS) -# Copyright 2022-2023 Xiqiang Liu - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -GEODATA - -Geospatial Data Collection and "Pre-Analysis" Tools -""" - -import calendar -import glob -import logging -import os -import tempfile -import zipfile -from pathlib import Path -from typing import Iterable - -import numpy as np -import xarray as xr - -from ..config import era5_dir - -logger = logging.getLogger(__name__) -datadir = era5_dir - -try: - import cdsapi - - has_cdsapi = True -except ImportError: - has_cdsapi = False - -# Model and Projection Settings -projection = "latlong" -L137_LEVELS = range(131, 138) - - -def _rename_and_clean_coords(ds: xr.Dataset, add_lon_lat: bool = True): - """Rename 'lon'/'longitude' and 'lat'/'latitude' columns to 'x' and 'y' - - Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. - - Args: - ds (xarray.Dataset): Dataset to rename - add_lon_lat (bool, optional): Add lon/lat columns. Defaults to True. - - Returns: - xarray.Dataset: Dataset with renamed coordinates - """ - - # Rename latitude / lat -> y, longitude / lon -> x - if "latitude" in list(ds.coords): - ds = ds.rename({"latitude": "y"}) - if "longitude" in list(ds.coords): - ds = ds.rename({"longitude": "x"}) - if "lat" in list(ds.coords): - ds = ds.rename({"lat": "y"}) - if "lon" in list(ds.coords): - ds = ds.rename({"lon": "x"}) - - if add_lon_lat: - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - - return ds - - -def api_complete( - toDownload: Iterable, - bounds: Iterable, - download_vars: Iterable, - product: str, - product_type: str, - downloadedFiles: list, -): - """Sample request: - - c.retrieve('reanalysis-era5-complete', { - 'date' : '20130101', - 'levelist': '1/10/100/137', - 'levtype' : 'ml', - 'param' : '130, # Full information at https://apps.ecmwf.int/codes/grib/param-db/ - 'stream' : 'oper', # Denotes ERA5. Ensemble members are selected by 'enda' - 'time' : '00/to/23/by/6', - 'type' : 'an', - 'grid' : '1.0/1.0', - 'format' : 'netcdf', - }, 'save_path.nc') # Output file. Adapt as you wish. - """ - - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - filepath = Path(f[1]) - filepath.parent.mkdir(parents=True, exist_ok=True) - - query_year = str(f[2]) - query_month = f"{f[3]:02d}" - - full_request = { - "date": f"{query_year}{query_month}01/to/{query_year}{query_month}{calendar.monthrange(f[2], f[3])[1]}", - "levelist": "/".join(str(i) for i in L137_LEVELS), - "levtype": "ml", - "param": "/".join(str(var) for var in download_vars), - "stream": "oper", - "time": "00/to/23", - "type": "an", - "grid": "0.25/0.25", # NOTE: We want the highest resolution possible - "format": "netcdf", - } - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["param"]), - f, - ) - - if not bounds: - full_result.download(f[1]) - else: - # NOTE: Raw MARS request doesn't support bounding box, so we need to - # subset the data after downloading - with tempfile.NamedTemporaryFile(suffix=".nc") as tmpfile: - full_result.download(tmpfile.name) - with xr.open_dataset(tmpfile.name, chunks="auto") as ds: - ds = ds.sel( - longitude=slice(*sorted([bounds[0], bounds[2]])), - latitude=slice( - *sorted([bounds[1], bounds[3]], reverse=True) - ), - ) - ds.to_netcdf(f[1]) - - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def api_hourly_era5( - toDownload: list, bounds, download_vars, product, product_type, downloadedFiles -): - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - print(f) - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - - ## for each file in self.todownload - need to then reextract year month in order to make query - query_year = str(f[2]) - query_month = str(f[3]) if len(str(f[3])) == 2 else "0" + str(f[3]) - - # 2. Full data file - full_request = { - "product_type": product_type, - "format": "netcdf", - "year": query_year, - "month": query_month, - "day": [f"{d:02d}" for d in range(1, 32)], - "time": [f"{h:02d}:00" for h in range(0, 24)], - "variable": download_vars, - } - - if bounds is not None: - # NOTE: cdsapi uses (long2, lat2, long1, lat1) format - full_request["area"] = bounds[::-1] - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["variable"]), - f, - ) - - if full_result.content_type == "application/zip": - logger.info( - "Multiple files found with request. Additional unzipping/preprocessing needed." - ) - - with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) - with zipfile.ZipFile( - os.path.join(tempdir, "download.zip"), "r" - ) as zip_ref: - zip_ref.extractall(tempdir) - - with xr.open_mfdataset( - [ - os.path.join(tempdir, f) - for f in os.listdir(tempdir) - if f.endswith(".nc") - ] - ) as ds: - ds.to_netcdf(f[1]) - - logger.info("Preprocessing complete with zipfile") - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - else: - full_result.download(f[1]) - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def api_monthly_era5( - toDownload, bounds, download_vars, product, product_type, downloadedFiles -): - if not has_cdsapi: - raise RuntimeError( - "Need installed cdsapi python package available from " - "https://cds.climate.copernicus.eu/api-how-to" - ) - - if len(toDownload) == 0: - logger.info("All ERA5 files for this dataset have been downloaded.") - else: - logger.info("Preparing to download %s files.", str(len(toDownload))) - - for f in toDownload: - print(f) - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - - ## for each file in self.todownload - need to then reextract year month in order to make query - query_year = str(f[2]) - query_month = str(f[3]) if len(str(f[3])) == 2 else "0" + str(f[3]) - - # 2. Full data file - full_request = { - "product_type": product_type, - "format": "netcdf", - "year": query_year, - "month": query_month, - "time": "00:00", - "variable": download_vars, - } - - if bounds is not None: - # cdsapi uses (long2, lat2, long1, lat1) format - full_request["area"] = bounds[::-1] - - full_result = cdsapi.Client().retrieve(product, full_request) - - logger.info( - "Downloading metadata request for %s variables to %s", - len(full_request["variable"]), - f, - ) - - if full_result.content_type == "application/zip": - logger.info( - "Multiple files found with request. Additional unzipping/preprocessing needed." - ) - - with tempfile.TemporaryDirectory() as tempdir: - full_result.download(os.path.join(tempdir, "download.zip")) - with zipfile.ZipFile( - os.path.join(tempdir, "download.zip"), "r" - ) as zip_ref: - zip_ref.extractall(tempdir) - - with xr.open_mfdataset( - [ - os.path.join(tempdir, f) - for f in os.listdir(tempdir) - if f.endswith(".nc") - ] - ) as ds: - ds.to_netcdf(f[1]) - - logger.info("Preprocessing complete with zipfile") - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - else: - full_result.download(f[1]) - logger.info("Successfully downloaded to %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def _add_height(ds): - """Convert geopotential 'z' to geopotential height following [1] - - References - ---------- - [1] ERA5: surface elevation and orography, retrieved: 10.02.2019 - https://confluence.ecmwf.int/display/CKB/ERA5%3A+surface+elevation+and+orography - - """ - g0 = 9.80665 - z = ds["z"] - if "time" in z.coords: - z = z.isel(time=0, drop=True) - ds["height"] = z / g0 - ds = ds.drop("z") - return ds - - -def convert_and_subset_lons_lats_era5(ds, xs, ys): - # Rename geographic dimensions to x,y - # Subset x,y according to xs, ys (subset_x_y_era5) - - # Rename lat and lon - ds = _rename_and_clean_coords(ds) - - # Longitudes should go from -180. to +180. - if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): - ds = xr.concat( - [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" - ) - ds = ds.assign_coords( - x=np.where( - ds.coords["x"].values <= 180, - ds.coords["x"].values, - ds.coords["x"].values - 360.0, - ) - ) - # Subset x and y - ds = subset_x_y_era5(ds, xs, ys) - - return ds - - -def subset_x_y_era5(ds, xs, ys): - # Subset x,y according to xs, ys - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(y=ys) - ds = ds.sel(x=xs) - - return ds - - -def prepare_meta_era5(xs, ys, year, month, template, module, **kwargs): - # Reference of the quantities - # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation - # Geopotential is aka Orography in the CDS: - # https://confluence.ecmwf.int/pages/viewpage.action?pageId=78296105 - - fns = glob.iglob(template.format(year=year, month=month)) - - try: - with xr.open_mfdataset(fns, combine="by_coords") as ds0: - ds = ds0.coords.to_dataset() - ds = convert_and_subset_lons_lats_era5(ds, xs, ys) - meta = ds.load() - except Exception as e: - logger.exception("Error when preparing for cutout: %s", e.args[0]) - raise e - return meta - - -def prepare_month_era5(fn, year, month, xs, ys): - # Reference of the quantities - # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation - # (shortName) | (name) | (paramId) - # tisr | TOA incident solar radiation | 212 - # ssrd | Surface Solar Rad Downwards | 169 - # ssr | Surface net Solar Radiation | 176 - # fdir | Total sky direct solar radiation at surface | 228021 - # ro | Runoff | 205 - # 2t | 2 metre temperature | 167 - # sp | Surface pressure | 134 - # stl4 | Soil temperature level 4 | 236 - # fsr | Forecast surface roughnes | 244 - - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = _add_height(ds) - ds = subset_x_y_era5(ds, xs, ys) - - ds = ds.rename({"fdir": "influx_direct", "tisr": "influx_toa"}) - with np.errstate(divide="ignore", invalid="ignore"): - ds["albedo"] = ( - ((ds["ssrd"] - ds["ssr"]) / ds["ssrd"]) - .fillna(0.0) - .assign_attrs(units="(0 - 1)", long_name="Albedo") - ) - ds["influx_diffuse"] = (ds["ssrd"] - ds["influx_direct"]).assign_attrs( - units="J m**-2", long_name="Surface diffuse solar radiation downwards" - ) - ds = ds.drop(["ssrd", "ssr"]) - - # Convert from energy to power J m**-2 -> W m**-2 and clip negative fluxes - for a in ("influx_direct", "influx_diffuse", "influx_toa"): - ds[a] = ds[a].clip(min=0.0) / (60.0 * 60.0) - ds[a].attrs["units"] = "W m**-2" - - ds["wnd100m"] = np.sqrt(ds["u100"] ** 2 + ds["v100"] ** 2).assign_attrs( - units=ds["u100"].attrs["units"], long_name="100 metre wind speed" - ) - ds = ds.drop(["u100", "v100"]) - - ds = ds.rename( - { - "ro": "runoff", - "t2m": "temperature", - "sp": "pressure", - "stl4": "soil temperature", - "fsr": "roughness", - } - ) - - # New ERA5 format for hourly datasets - # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 - # TODO: We can remove this if we refactor geodata's convert module in the future - if "valid_time" in ds.coords: - ds = ds.rename({"valid_time": "time"}) - - ds["runoff"] = ds["runoff"].clip(min=0.0) - yield (year, month), ds - - -def tasks_monthly_era5(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info(list(yearmonths)) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format(year=year, month=month), - ) - for year, month in yearmonths - ] - - -def prepare_3d_era5(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = subset_x_y_era5(ds, xs, ys) - - # New ERA5 format for hourly datasets - # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 - # TODO: We can remove this if we refactor geodata's convert module in the future - if "valid_time" in ds.coords: - ds = ds.rename({"valid_time": "time"}) - - yield (year, month), ds - - -weather_data_config = { - "wind_solar_hourly": dict( - api_func=api_hourly_era5, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_month_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_hourly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_hourly.nc"), - product="reanalysis-era5-single-levels", - product_type="reanalysis", - keywords=[ - "100m_u_component_of_wind", - "100m_v_component_of_wind", - "2m_temperature", - "runoff", - "soil_temperature_level_4", - "surface_net_solar_radiation", - "surface_pressure", - "surface_solar_radiation_downwards", - "toa_incident_solar_radiation", - "total_sky_direct_solar_radiation_at_surface", - "forecast_surface_roughness", - "geopotential", - ], - variables=[ - "u100", - "v100", - "t2m", - "ro", - "stl4", - "ssr", - "sp", - "ssrd", - "tisr", - "fdir", - "fsr", - "z", - ], - ), - "wind_3d_hourly": dict( - api_func=api_complete, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_3d_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_3d_hourly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_3d_hourly.nc"), - product="reanalysis-era5-complete", - product_type="reanalysis", - keywords=[131, 132], - variables=["u", "v"], - ), - "wind_solar_monthly": dict( - api_func=api_monthly_era5, - file_granularity="monthly", - tasks_func=tasks_monthly_era5, - meta_prepare_func=prepare_meta_era5, - prepare_func=prepare_month_era5, - template=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_monthly.nc"), - fn=os.path.join(era5_dir, "{year}/{month:0>2}/wind_solar_monthly.nc"), - product="reanalysis-era5-single-levels-monthly-means", - product_type="monthly_averaged_reanalysis", - keywords=[ - "100m_u_component_of_wind", - "100m_v_component_of_wind", - "2m_temperature", - "runoff", - "soil_temperature_level_4", - "surface_net_solar_radiation", - "surface_pressure", - "surface_solar_radiation_downwards", - "toa_incident_solar_radiation", - "total_sky_direct_solar_radiation_at_surface", - "forecast_surface_roughness", - "geopotential", - ], - variables=[ - "u100", - "v100", - "t2m", - "ro", - "stl4", - "ssr", - "sp", - "ssrd", - "tisr", - "fdir", - "fsr", - "z", - ], - ), -} - -# No separate files for each day (would be coded in weather_data_config list, see merra2.py) -daily_files = False - -# Latitude direction stored -# South to north = True -# North to south = False -lat_direction = False - -# Spinup variable (necessary for MERRA) -spinup_var = False diff --git a/src/geodata/datasets/era5/__init__.py b/src/geodata/datasets/era5/__init__.py new file mode 100644 index 00000000..ca00fb2d --- /dev/null +++ b/src/geodata/datasets/era5/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import hourly, monthly + +__all__ = ["hourly", "monthly"] diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py new file mode 100644 index 00000000..ecc1ef11 --- /dev/null +++ b/src/geodata/datasets/era5/_base.py @@ -0,0 +1,163 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os + +import numpy as np +import xarray as xr + +from ...types import CoordRange, PathLike +from .._base import BaseDataset + +logger = logging.getLogger(__name__) + +try: + import cdsapi +except ImportError: + logger.warning( + "cdsapi is not installed. You will not be able to download ERA5 data." + "Please install it with `pip install 'geodata-re[download]'." + ) + cdsapi = None + + +def _convert_and_subset_lons_lats_era5(ds: xr.Dataset, xs: slice, ys: slice): + # Longitudes should go from -180. to +180. + if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): + ds = xr.concat( + [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" + ) + ds = ds.assign_coords( + x=np.where( + ds.coords["x"].values <= 180, + ds.coords["x"].values, + ds.coords["x"].values - 360.0, + ) + ) + + # Subset x and y + return _subset_x_y_era5(ds, xs, ys) + + +def _subset_x_y_era5(ds: xr.Dataset, xs: slice, ys: slice): + # Subset x,y according to xs, ys + + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + ds = ds.sel(x=xs) + + return ds + + +class ERA5BaseDataset(BaseDataset): + """ERA5BaseDataset is a class that encaps a dataset from the ERA5 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + module = "era5" + projection = "latlong" + lat_direction = False + + def _extra_setup(self, **kwargs): + self.logger = logging.getLogger(__name__.replace("._base", ".client")) + self.client = cdsapi.Client( + info_callback=self.logger.info, + error_callback=self.logger.error, + debug_callback=self.logger.debug, + warning_callback=self.logger.warning, + ) + + @classmethod + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + # Reference of the quantities + # https://confluence.ecmwf.int/display/CKB/ERA5+data+documentation + # Geopotential is aka Orography in the CDS: + # https://confluence.ecmwf.int/pages/viewpage.action?pageId=78296105 + + with xr.open_mfdataset(cls._get_path(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _convert_and_subset_lons_lats_era5(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + prepare_func: callable, + **meta_attrs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + fn = meta_attrs["fn"] + + logger.info(yearmonths) + logger.info(list(yearmonths)) + + return [ + dict( + prepare_func=prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format(year=year, month=month), + ) + for year, month in yearmonths + ] + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month.""" + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn) as ds: + logger.info("Opening %s", fn) + ds = _subset_x_y_era5(ds, xs, ys) + + # New ERA5 format for hourly datasets + # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 + # TODO: We can remove this if we refactor geodata's convert module in the future + if "valid_time" in ds.coords: + ds = ds.rename({"valid_time": "time"}) + + yield (year, month), ds + + def _dataset_postprocess(self, ds, **kwargs): + return super()._dataset_postprocess(ds, **kwargs) diff --git a/src/geodata/datasets/era5/hourly/__init__.py b/src/geodata/datasets/era5/hourly/__init__.py new file mode 100644 index 00000000..b76666fc --- /dev/null +++ b/src/geodata/datasets/era5/hourly/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_3d import ERA5Wind3DHourlyDataset +from .wind_solar import ERA5WindSolarHourlyDataset + +__all__ = ["ERA5WindSolarHourlyDataset", "ERA5Wind3DHourlyDataset"] diff --git a/src/geodata/datasets/era5/hourly/wind_3d.py b/src/geodata/datasets/era5/hourly/wind_3d.py new file mode 100644 index 00000000..95aeef4f --- /dev/null +++ b/src/geodata/datasets/era5/hourly/wind_3d.py @@ -0,0 +1,116 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import pprint +import tempfile +from pathlib import Path + +import xarray as xr + +from ..._base import AtomicDataset +from .._base import ERA5BaseDataset + +logger = logging.getLogger(__name__) + + +class ERA5Wind3DHourlyDataset(ERA5BaseDataset): + """ERA5Wind3DHourlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind information. + This dataset is stored in hourly intervals. + """ + + weather_config = "wind_3d_hourly" + frequency = "daily" + L137_LEVELS = range(131, 138) + + # Information that are needed for ERA5's API request + variables = {"u": "131", "v": "132"} + product = "reanalysis-era5-complete" + + def _download_file(self, file: AtomicDataset): + """Sample request for reference: + + c.retrieve('reanalysis-era5-complete', { + 'date' : '20130101/to/20130131', + 'levelist': '1/10/100/137', + 'levtype' : 'ml', + 'param' : '130, # Full information at https://apps.ecmwf.int/codes/grib/param-db/ + 'stream' : 'oper', # Denotes ERA5. Ensemble members are selected by 'enda' + 'time' : '00/to/23/by/6', + 'type' : 'an', + 'grid' : '1.0/1.0', + 'format' : 'netcdf', + }, 'save_path.nc') # Output file. Adapt as you wish. + """ + + year: int = file.year + month: int = file.month + day: int = file.day + save_path: Path = file.path + + full_request = { + "date": f"{year}{month:02d}{day:02d}", + "levelist": "/".join([str(level) for level in self.L137_LEVELS]), + "levtype": "ml", + "param": "/".join(self.variables.values()), + "stream": "oper", + "time": "00/to/23/by/1", + "type": "an", + "grid": "0.25/0.25", # NOTE: We want the highest resolution possible + "format": "netcdf", + } + + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + + full_result = self.client.retrieve(self.product, full_request) + + if not self.bounds: + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(save_path) + logger.info("File downloaded: %s", save_path) + return + except Exception as e: + logger.error("Download failed: %s", e) + if _count == 3: + raise + + # NOTE: Raw MARS request doesn't support bounding box, so we need to + # subset the data after downloading + with tempfile.NamedTemporaryFile(suffix=".nc") as tmpfile: + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(tmpfile.name) + logger.info("File downloaded: %s", save_path) + break + except Exception as e: + logger.error("Download failed: %s", e) + if _count == 3: + raise + with xr.open_dataset(tmpfile.name, chunks="auto") as ds: + ds = ds.sel( + longitude=slice(*sorted([self.bounds[0], self.bounds[2]])), + latitude=slice( + *sorted([self.bounds[1], self.bounds[3]], reverse=True) + ), + ) + ds.to_netcdf(save_path) + + logger.info("File downloaded: %s", save_path) diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/hourly/wind_solar.py new file mode 100644 index 00000000..8b139951 --- /dev/null +++ b/src/geodata/datasets/era5/hourly/wind_solar.py @@ -0,0 +1,132 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import pprint +import tempfile +import zipfile +from pathlib import Path + +import xarray as xr + +from ..._base import AtomicDataset +from .._base import ERA5BaseDataset + +logger = logging.getLogger(__name__) + + +class ERA5WindSolarHourlyDataset(ERA5BaseDataset): + """ERA5WindSolarHourlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind and solar + information. This dataset is stored in hourly intervals. + + The ERA5 dataset is a reanalysis dataset that provides a comprehensive + record of the Earth's climate. It is produced by the European Centre for + Medium-Range Weather Forecasts (ECMWF) and is available from 1980 to + present. + + Note: + - The specific variables that are downloaded are: + - 100m_u_component_of_wind + - 100m_v_component_of_wind + - 2m_temperature + - 2m_dewpoint_temperature + - runoff + - soil_temperature_level_4 + - surface_net_solar_radiation + - surface_pressure + - surface_solar_radiation_downwards + - toa_incident_solar_radiation + - total_sky_direct_solar_radiation_at_surface + - forecast_surface_roughness + - geopotential + """ + + weather_config = "wind_solar_hourly" + + # Information that is needed for ERA5's API request + variables = { + "100m_u_component_of_wind": "u100", + "100m_v_component_of_wind": "v100", + "2m_temperature": "t2m", + "2m_dewpoint_temperature": "d2m", + "runoff": "ro", + "soil_temperature_level_4": "stl4", + "surface_net_solar_radiation": "ssr", + "surface_pressure": "sp", + "surface_solar_radiation_downwards": "ssrd", + "toa_incident_solar_radiation": "tisr", + "total_sky_direct_solar_radiation_at_surface": "fdir", + "forecast_surface_roughness": "fsr", + "geopotential": "z", + } + product = "reanalysis-era5-single-levels" + product_type = "reanalysis" + + def _download_file(self, file: AtomicDataset): + year: int = file.year + month: int = file.month + save_path: Path = file.path + + full_request = { + "product_type": self.product_type, + "format": "netcdf", + "variable": list(self.variables.keys()), + "year": year, + "month": month, + "day": [f"{d:02d}" for d in range(1, 32)], + "time": [f"{t:02d}:00" for t in range(0, 24)], + } + + if self.bounds is not None: + full_request["area"] = self.bounds[::-1] + + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + + full_result = self.client.retrieve(self.product, full_request) + if full_result.content_type == "application/zip": + logger.info( + "Multiple files found with request. Additional unzipping/preprocessing needed." + ) + + with tempfile.TemporaryDirectory() as tempdir: + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(os.path.join(tempdir, "download.zip")) + break + except Exception as e: + logger.error("Error downloading file: %s", e) + if _count == 3: + raise + + with zipfile.ZipFile( + os.path.join(tempdir, "download.zip"), "r" + ) as zip_ref: + zip_ref.extractall(tempdir) + + with xr.open_mfdataset( + [ + os.path.join(tempdir, f) + for f in os.listdir(tempdir) + if f.endswith(".nc") + ] + ) as ds: + ds.to_netcdf(save_path) + + logger.info("Preprocessing complete with zipfile") + logger.info("Successfully downloaded to %s", save_path) diff --git a/src/geodata/datasets/era5/monthly/__init__.py b/src/geodata/datasets/era5/monthly/__init__.py new file mode 100644 index 00000000..98d2dfe8 --- /dev/null +++ b/src/geodata/datasets/era5/monthly/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_solar import ERA5WindSolarMonthlyDataset + +__all__ = ["ERA5WindSolarMonthlyDataset"] diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/monthly/wind_solar.py new file mode 100644 index 00000000..8a523b48 --- /dev/null +++ b/src/geodata/datasets/era5/monthly/wind_solar.py @@ -0,0 +1,114 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import pprint +import tempfile +import zipfile +from pathlib import Path + +import xarray as xr + +from ..._base import AtomicDataset +from ..hourly.wind_solar import ERA5WindSolarHourlyDataset + +logger = logging.getLogger(__name__) + + +class ERA5WindSolarMonthlyDataset(ERA5WindSolarHourlyDataset): + """ERA5WindSolarMonthlyDataset is a class that handles the downloading, + preprocessing, and storing of the ERA5 dataset for wind and solar + information. This dataset is stored in monthly intervals. + + The ERA5 dataset is a reanalysis dataset that provides a comprehensive + record of the Earth's climate. It is produced by the European Centre for + Medium-Range Weather Forecasts (ECMWF) and is available from 1980 to + present. + + Note: + - The specific variables that are downloaded are: + - 100m_u_component_of_wind + - 100m_v_component_of_wind + - 2m_temperature + - 2m_dew_point_temperature + - runoff + - soil_temperature_level_4 + - surface_net_solar_radiation + - surface_pressure + - surface_solar_radiation_downwards + - toa_incident_solar_radiation + - total_sky_direct_solar_radiation_at_surface + - forecast_surface_roughness + - geopotential + """ + + weather_config = "wind_solar_monthly" + frequency = "monthly" + + def _download_file(self, file: AtomicDataset): + year: int = file.year + month: int = file.month + save_path: Path = file.path + + full_request = { + "product_type": self.product_type, + "format": "netcdf", + "variable": list(self.variables.keys()), + "year": year, + "month": month, + "day": [f"{d:02d}" for d in range(1, 32)], + "time": "00:00", + } + + if self.bounds is not None: + full_request["area"] = self.bounds[::-1] + + logger.debug("Full request for download: %s", pprint.pformat(full_request)) + + full_result = self.client.retrieve(self.product, full_request) + if full_result.content_type == "application/zip": + logger.info( + "Multiple files found with request. Additional unzipping/preprocessing needed." + ) + + with tempfile.TemporaryDirectory() as tempdir: + _count = 0 + for _ in range(3): + try: + _count += 1 + full_result.download(os.path.join(tempdir, "download.zip")) + break + except Exception as e: + logger.error("Error downloading file: %s", e) + if _count == 3: + raise + + with zipfile.ZipFile( + os.path.join(tempdir, "download.zip"), "r" + ) as zip_ref: + zip_ref.extractall(tempdir) + + with xr.open_mfdataset( + [ + os.path.join(tempdir, f) + for f in os.listdir(tempdir) + if f.endswith(".nc") + ] + ) as ds: + ds.to_netcdf(save_path) + + logger.info("Preprocessing complete with zipfile") + logger.info("Successfully downloaded to %s", save_path) diff --git a/src/geodata/datasets/hrrr/__init__.py b/src/geodata/datasets/hrrr/__init__.py new file mode 100644 index 00000000..334f2004 --- /dev/null +++ b/src/geodata/datasets/hrrr/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import hourly + +__all__ = ["hourly"] diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py new file mode 100644 index 00000000..1525c5d1 --- /dev/null +++ b/src/geodata/datasets/hrrr/_base.py @@ -0,0 +1,140 @@ +# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import logging +import os.path as osp +import tempfile + +import herbie +import numpy as np +import pandas as pd +import xarray as xr + +from ...types import CoordRange +from .._base import BaseDataset + +logger = logging.getLogger(__name__) + + +def _subset_x_y(ds: xr.Dataset, xs: slice, ys: slice): + # Subset x,y according to xs, ys + + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + ds = ds.sel(x=xs) + + return ds + + +class HRRRBaseDataset(BaseDataset): + """HRRRBaseDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + module = "hrrr" + projection = "latlong" + frequency = "daily" + _priority = ["google", "aws", "azure"] + + lat_direction = True + + def _extra_setup(self, **kwargs): + self._herbie_save_dir = tempfile.TemporaryDirectory() + + def __del__(self): + self._herbie_save_dir.cleanup() + + def _preprocess_individual_herbie( + self, h: herbie.Herbie, search: str, hour: pd.DatetimeIndex + ): + tmp_ds = h.xarray(search, remove_grib=False) + tmp_ds.to_netcdf(osp.join(self._herbie_save_dir.name, f"{hour}_{search}.nc")) + tmp_ds.close() + + return osp.join(self._herbie_save_dir.name, f"{hour}_{search}.nc") + + def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): + # Because certain hours are missing, we need to reindex the dataset + # to include all hours in the range + + logger.debug("Reindexing dataset to include all hours in the range") + dt: np.datetime64 = ( + ds["time"].values[0].astype("datetime64[D]").astype("datetime64[m]") + ) + ds = ds.reindex(time=pd.date_range(dt, periods=24, freq="h")).ffill(dim="time") + + # NOTE: For some reasons, HRRR's longitude is in the range of [0, 360] + # instead of [-180, 180]. We need to put it back to [-180, 180]. + logger.debug("Fixing longitude range") + ds["x"] = (ds["x"] % 360 + 540) % 360 - 180 + + ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + del ds["valid_time"] + + return ds + + @classmethod + def meta_prepare_func(cls, xs: slice, ys: slice, year: int, month: int, **kwargs): + with xr.open_mfdataset(cls._get_files(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _subset_x_y(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + **kwargs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + + return [ + dict( + prepare_func=cls.prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=cls._get_files(year, month, **kwargs), + ) + for year, month in yearmonths + ] + + @staticmethod + def prepare_func(fn, year, month, xs, ys, **kwargs): + if isinstance(fn, str) and not osp.exists(fn): + return + if isinstance(fn, list) and not all(osp.isfile(f) for f in fn): + return + + with xr.open_mfdataset(fn) as ds: + logger.info("Opening %s", fn) + + yield (year, month), _subset_x_y(ds, xs, ys) diff --git a/src/geodata/datasets/hrrr/hourly/__init__.py b/src/geodata/datasets/hrrr/hourly/__init__.py new file mode 100644 index 00000000..5b6188c6 --- /dev/null +++ b/src/geodata/datasets/hrrr/hourly/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .wind_3d import HRRR3DWindHourlyDataset +from .wind_solar import HRRRHourlyDataset + +__all__ = ["HRRRHourlyDataset", "HRRR3DWindHourlyDataset"] diff --git a/src/geodata/datasets/hrrr/hourly/wind_3d.py b/src/geodata/datasets/hrrr/hourly/wind_3d.py new file mode 100644 index 00000000..a70441ff --- /dev/null +++ b/src/geodata/datasets/hrrr/hourly/wind_3d.py @@ -0,0 +1,139 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import multiprocessing as mp +import os +from datetime import datetime, timedelta + +import pandas as pd +import xarray as xr +from herbie import FastHerbie + +from ....logging import redirect_stdout_to_logger +from ..._base import AtomicDataset +from .._base import HRRRBaseDataset + +logger = logging.getLogger(__name__) + + +class HRRR3DWindHourlyDataset(HRRRBaseDataset): + """HRRR3DWindHourlyDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + + Variables: + - u: zonal wind component + - v: meridional wind component + - gh: geopotential height + + Important Coordinates: + - time: time of the observation + - level: hybrid pressure level + - y: latitude + - x: longitude + """ + + weather_config = "hrrr_wind_3d_hourly" + product = "nat" # Use "nat" product for 3D data + + def _download_file(self, file: AtomicDataset): + year, month, day = file.year, file.month, file.day + + start = datetime(year, month, day) + end = start + timedelta(days=1) + date_range = pd.date_range(start, end, freq="h", inclusive="left") + + with redirect_stdout_to_logger(logger, logging.INFO): + logger.info(f"Downloading HRRR wind data in bulk for {year}/{month}") + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + fh.download(":[UV]GRD:[1,8]0 m") + fh.download(":[UV]GRD:[1234] hybrid level") + fh.download(":HGT:[1234] hybrid level") + + fh.xarray(":[UV]GRD:10 m", remove_grib=False).rename( + {"u10": "u", "v10": "v"} + ).to_netcdf(os.path.join(self._herbie_save_dir.name, "uv_10.nc")) + + fh.xarray(":[UV]GRD:80 m", remove_grib=False).to_netcdf( + os.path.join(self._herbie_save_dir.name, "uv_80.nc") + ) + fh.xarray(":[UV]GRD:[1234] hybrid level", remove_grib=False).to_netcdf( + os.path.join(self._herbie_save_dir.name, "uv_hybrid.nc") + ) + fh.xarray(":HGT:[1234] hybrid level", remove_grib=False).to_netcdf( + os.path.join(self._herbie_save_dir.name, "hgt_hybrid.nc") + ) + + uv_10 = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_10.nc"), chunks="auto" + ) + uv_80 = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_80.nc"), chunks="auto" + ) + uv_hybrid = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "uv_hybrid.nc"), chunks="auto" + ) + hgt_hybrid = xr.open_dataset( + os.path.join(self._herbie_save_dir.name, "hgt_hybrid.nc"), chunks="auto" + ) + + ds: xr.Dataset = xr.concat([uv_10, uv_80], dim="heightAboveGround") + heights = ds["heightAboveGround"].broadcast_like(ds["u"]) + + ds["u"] = xr.concat( + [ds["u"].rename({"heightAboveGround": "hybrid"}), uv_hybrid["u"]], + dim="hybrid", + ) + ds["v"] = xr.concat( + [ds["v"].rename({"heightAboveGround": "hybrid"}), uv_hybrid["v"]], + dim="hybrid", + ) + del ds["heightAboveGround"] + + ds["gh"] = xr.concat( + [heights.rename({"heightAboveGround": "hybrid"}), hgt_hybrid["gh"]], + dim="hybrid", + ).astype("float32") + ds = ds.rename({"hybrid": "level"}).sortby("level") + + ds["level"].values[-2:] = [-1, -2] + ds["level"] = ds["level"].astype("int8") + + ds = ds.sortby("level") + + try: + del ds.attrs["search"] + del ds.attrs["local_grib"] + del ds.attrs["remote_grib"] + del ds["lon"] + del ds["lat"] + except KeyError: + pass + + ds.to_netcdf(file.path) + + # NOTE: Flush temporary FastHerbie save directory to save space, since we no + # longer need the raw downloaded files + self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/hrrr/hourly/wind_solar.py b/src/geodata/datasets/hrrr/hourly/wind_solar.py new file mode 100644 index 00000000..e08c5ce0 --- /dev/null +++ b/src/geodata/datasets/hrrr/hourly/wind_solar.py @@ -0,0 +1,180 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import multiprocessing as mp + +import pandas as pd +import xarray as xr +from herbie import FastHerbie, Herbie + +from ....logging import redirect_stdout_to_logger +from ..._base import AtomicDataset +from .._base import HRRRBaseDataset + +logger = logging.getLogger(__name__) + + +class HRRRHourlyDataset(HRRRBaseDataset): + """HRRRHourlyDataset is a class that encaps a dataset from the HRRR + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + + The HRRR dataset is a high-resolution weather forecast model that provides + hourly data for the United States. This dataset is useful for a variety of + applications, including renewable energy forecasting, weather prediction, + and climate research. + + This class provides a simple interface for downloading and processing the + HRRR dataset. It allows users to specify the years and months of interest, + as well as the variables they wish to download. + """ + + weather_config = "hrrr_wind_solar_hourly" + product = "sfc" + + def _download_file(self, file: AtomicDataset): + year, month, day = file.year, file.month, file.day + + date_range = pd.date_range( + f"{year}-{month}-{day}", + f"{year}-{month}-{day+1}", + freq="h", + inclusive="left", + ) + + fh = FastHerbie( + date_range, + model=self.module, + product=self.product, + max_threads=mp.cpu_count() * 2, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + with redirect_stdout_to_logger(logger, logging.INFO): + fh.download(":[UV]GRD:[1,8]0 m") + fh.download(":TMP:2 m") + fh.download(":..WRF:surface") + fh.download(":TCDC:entire atmosphere") + fh.download(":LCDC:low cloud layer") + fh.download(":MCDC:middle cloud layer") + fh.download(":HCDC:high cloud layer") + + uv_10 = [] + uv_80 = [] + tmp_2 = [] + wrfs = [] + tcdc = [] + lcdc = [] + mcdc = [] + hcdc = [] + + for hour in date_range: + h = Herbie( + hour, + model=self.module, + product=self.product, + save_dir=self._herbie_save_dir.name, + priority=self._priority, + ) + + try: + uv_10.append( + self._preprocess_individual_herbie(h, "[UV]GRD:10 m", hour) + ) + uv_80.append( + self._preprocess_individual_herbie(h, "[UV]GRD:80 m", hour) + ) + tmp_2.append( + self._preprocess_individual_herbie(h, ":TMP:2 m", hour) + ) + wrfs.append( + self._preprocess_individual_herbie(h, ":..WRF:surface", hour) + ) + tcdc.append( + self._preprocess_individual_herbie( + h, ":TCDC:entire atmosphere", hour + ) + ) + lcdc.append( + self._preprocess_individual_herbie( + h, ":LCDC:low cloud layer", hour + ) + ) + mcdc.append( + self._preprocess_individual_herbie( + h, ":MCDC:middle cloud layer", hour + ) + ) + hcdc.append( + self._preprocess_individual_herbie( + h, ":HCDC:high cloud layer", hour + ) + ) + + except ValueError: + logger.warning(f"No data found for {hour}, skipping.") + + # First concat to daily files + + uv_10: xr.Dataset = xr.open_mfdataset( + uv_10, concat_dim="time", chunks="auto", combine="nested" + ) + uv_80: xr.Dataset = xr.open_mfdataset( + uv_80, concat_dim="time", chunks="auto", combine="nested" + ).rename({"u": "u80", "v": "v80"}) + tmp_2: xr.Dataset = xr.open_mfdataset( + tmp_2, concat_dim="time", chunks="auto", combine="nested" + ) + wrfs: xr.Dataset = xr.open_mfdataset( + wrfs, concat_dim="time", chunks="auto", combine="nested" + ) + tcdc: xr.Dataset = xr.open_mfdataset( + tcdc, concat_dim="time", chunks="auto", combine="nested" + ) + lcdc: xr.Dataset = xr.open_mfdataset( + lcdc, concat_dim="time", chunks="auto", combine="nested" + ) + mcdc: xr.Dataset = xr.open_mfdataset( + mcdc, concat_dim="time", chunks="auto", combine="nested" + ) + hcdc: xr.Dataset = xr.open_mfdataset( + hcdc, concat_dim="time", chunks="auto", combine="nested" + ) + + ds = xr.merge( + [uv_10, uv_80, tmp_2, wrfs, lcdc, tcdc, mcdc, hcdc], compat="override" + ) + + try: + del ds["heightAboveGround"] + del ds.attrs["search"] + del ds.attrs["local_grib"] + del ds.attrs["remote_grib"] + del ds.coords["gribfile_projection"] + del ds.coords["surface"] + del ds.coords["lowCloudLayer"] + del ds.coords["middleCloudLayer"] + del ds.coords["highCloudLayer"] + del ds.coords["atmosphere"] + except KeyError: + pass + + ds.to_netcdf(file.path) + + # NOTE: Flush temporary FastHerbie save directory to save space, since we no + # longer need the raw downloaded files + self._herbie_save_dir.cleanup() diff --git a/src/geodata/datasets/merra2.py b/src/geodata/datasets/merra2.py deleted file mode 100644 index 014b09f1..00000000 --- a/src/geodata/datasets/merra2.py +++ /dev/null @@ -1,612 +0,0 @@ -# Copyright 2020 Michael Davidson (UCSD), William Honaker. - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -GEODATA - -Geospatial Data Collection and "Pre-Analysis" Tools -""" - -import glob -import os -from calendar import monthrange -from tempfile import mkstemp - -import numpy as np -import requests -import xarray as xr -from requests.exceptions import HTTPError -from tqdm.contrib.logging import tqdm_logging_redirect - -from ..config import merra2_dir -from ..logging import logger - -datadir = merra2_dir - -# Model and Projection Settings -projection = "latlong" - - -def convert_and_subset_lons_lats_merra2(ds, xs, ys): - # Rename geographic dimensions to x,y - # Subset x,y according to xs, ys - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(lat=ys) - - # Lons should go from -180. to +180. - if len(ds.coords["lon"].sel(lon=slice(xs.start + 360.0, xs.stop + 360.0))): - ds = xr.concat( - [ds.sel(lon=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(lon=xs)], - dim="lon", - ) - ds = ds.assign_coords( - lon=np.where( - ds.coords["lon"].values <= 180, - ds.coords["lon"].values, - ds.coords["lon"].values - 360.0, - ) - ) - else: - ds = ds.sel(lon=xs) - - ds = ds.rename({"lon": "x", "lat": "y"}) - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - return ds - - -def subset_x_y_merra2(ds, xs, ys): - # Subset x,y according to xs, ys - # Assumes convert_and_subset_lons_lats_merra2 already run - - if not isinstance(xs, slice): - first, second, last = np.asarray(xs)[[0, 1, -1]] - xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - if not isinstance(ys, slice): - first, second, last = np.asarray(ys)[[0, 1, -1]] - ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) - - ds = ds.sel(y=ys) - ds = ds.sel(x=xs) - - return ds - - -def _rename_and_clean_coords(ds, add_lon_lat=True): - """Rename 'longitude' and 'latitude' columns to 'x' and 'y' - - Optionally (add_lon_lat, default:True) preserves latitude and longitude columns as 'lat' and 'lon'. - """ - - ds = ds.rename({"lon": "x", "lat": "y"}) - if add_lon_lat: - ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) - return ds - - -def api_merra2(toDownload, fileGranularity, downloadedFiles): - if len(toDownload) == 0: - logger.info("All MERRA2 files for this dataset have been downloaded.") - else: - multi = bool(fileGranularity in ("daily_multiple", "monthly_multiple")) - - total = len(toDownload) * (len(toDownload[0]) - 2) if multi else len(toDownload) - with tqdm_logging_redirect( - loggers=[logger], total=total, dynamic_ncols=True - ) as pbar: - for f in toDownload: - error_files = [] - if multi: - fd, target = mkstemp(suffix=".nc4") - else: - fd = 0 - target = f[1] - os.makedirs(os.path.dirname(f[1]), exist_ok=True) - logger.info("Preparing API calls for %s", f[1]) - logger.info("Making request to %s", f[2]) - result = requests.get(f[2], timeout=30) - try: - result.raise_for_status() - with open(target, "wb") as fout: - fout.write(result.content) - except HTTPError as http_err: - logger.warning("HTTP error occurred: %s", http_err) # Python 3.6 - error_files.append(f[1]) - except Exception as err: # pylint: disable=broad-except - logger.warning("Other error occurred: %s", err) - error_files.append(f[1]) - pbar.update() - - if multi: - # ds_main = xr.open_dataset(target) - - temp_files = [[fd, target]] - for k in range(3, len(f)): - logger.info("Making request to %s", f[k]) - result = requests.get(f[k], timeout=30) - fd_temp, target_temp = mkstemp(suffix=".nc4") - temp_files.append([fd_temp, target_temp]) - try: - result.raise_for_status() - with open(target_temp, "wb") as fout: - fout.write(result.content) - except HTTPError as http_err: - logger.warning( - "HTTP error occurred: %s", http_err - ) # Python 3.6 - error_files.append(f[k]) - except Exception as err: # pylint: disable=broad-except - logger.warning("Other error occurred: %s", err) - error_files.append(f[k]) - # ds_toadd = xr.open_dataset(target_temp) - # ds_main = xr.merge([ds_main, ds_toadd]) - # os.close(fd_temp) - # os.unlink(target_temp) - pbar.update() - - ds_main = xr.open_mfdataset( - [fn[1] for fn in temp_files], combine="by_coords" - ) - ds_main.to_netcdf(f[1]) - ds_main.close() - # ds_toadd.close() # close last xr open file - - # close and clear temp files - # os.close(fd) - # os.unlink(target) - for tf in temp_files: - os.close(tf[0]) - os.unlink(tf[1]) - - if len(error_files) > 0: - logger.warning("Unsuccessful download for %s", error_files) - else: - logger.info("Successfully downloaded data for %s", f[1]) - downloadedFiles.append((f[0], f[1])) - - -def prepare_meta_merra2(xs, ys, year, month, template, module, **params): - # Load dataset into metadata - - # fn = next(glob.iglob(template.format(year=year, month=month))) - # with xr.open_dataset(fn) as ds: - # ds = ds.coords.to_dataset() - # ds = convert_and_subset_lons_lats_merra2(ds, xs, ys) - # meta = ds.load() - - # Set spinup variable (see MERRA2 documentation, p. 13) - spinup = spinup_year(year, month) - - fns = glob.iglob(template.format(year=year, month=month, spinup=spinup)) - with xr.open_mfdataset(fns, combine="by_coords") as ds: - ds = ds.coords.to_dataset() - ds = convert_and_subset_lons_lats_merra2(ds, xs, ys) - meta = ds.load() - - return meta - - -def prepare_month_surface_flux(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - # some variable renaming - try: - # z0m=roughness - # wind variables not in wndXXm format - ds = ds.rename({"z0m": "roughness"}) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - - ds["wndlml"] = np.sqrt(ds["ulml"] ** 2 + ds["vlml"] ** 2).assign_attrs( - units=ds["ulml"].attrs["units"], long_name="LML wind speed" - ) - if "tlml" in list(ds.data_vars): - ds["temperature"] = ds["tlml"] - - yield (year, month), ds - - -def prepare_month_aerosol(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _rename_and_clean_coords(ds) - ds = subset_x_y_merra2(ds, xs, ys) - yield (year, month), ds - - -def prepare_dailymeans_surface_flux(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - # some variable renaming - try: - # z0m=roughness - # wind variables not in wndXXm format - ds = ds.rename({"t2mmean": "temperature", "tprecmax": "precipitation"}) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - - # ['HOURNORAIN', 'T2MMAX', 'T2MMEAN', 'T2MMIN', 'TPRECMAX'] - - yield (year, month), ds - - -def prepare_slv_radiation(fn, year, month, xs, ys): - if not os.path.isfile(fn): - return None - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = _rename_and_clean_coords(ds) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - - ds = subset_x_y_merra2(ds, xs, ys) - - # logger.info("Cutout dims: %s", ds.dims) - # logger.info("Cutout coords: %s", ds.coords) - try: - ds = ds.rename( - { - "albedo": "albedo", - "swgdn": "influx", - "swtdn": "influx_toa", - "t2m": "temperature", - } - ) - except Exception as e: - logger.warning("Unable to rename variables in %s. Exception: %s", fn, e) - yield (year, month), ds - - -## TODO def prepare_month_radiation -# with np.errstate(divide='ignore', invalid='ignore'): -# ds['albedo'] = (((ds['ssrd'] - ds['ssr'])/ds['ssrd']).fillna(0.) -# .assign_attrs(units='(0 - 1)', long_name='Albedo')) -# ds['influx_diffuse'] = ((ds['ssrd'] - ds['influx_direct']) -# .assign_attrs(units='J m**-2', -# long_name='Surface diffuse solar radiation downwards')) -# ds = ds.drop(['ssrd', 'ssr']) -# -# # Convert from energy to power J m**-2 -> W m**-2 and clip negative fluxes -# for a in ('influx_direct', 'influx_diffuse', 'influx_toa'): -# ds[a] = ds[a].clip(min=0.) / (60.*60.) -# ds[a].attrs['units'] = 'W m**-2' - - -def tasks_daily_merra2(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info( - [ - (year, month, day) - for year, month in yearmonths - for day in range(1, monthrange(year, month)[1] + 1, 1) - ] - ) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format( - year=year, month=month, day=day, spinup=spinup_year(year, month) - ), - ) - for year, month in yearmonths - for day in range(1, monthrange(year, month)[1] + 1, 1) - ] - - -def tasks_monthly_merra2(xs, ys, yearmonths, prepare_func, **meta_attrs): - if not isinstance(xs, slice): - xs = slice(*xs.values[[0, -1]]) - if not isinstance(ys, slice): - ys = slice(*ys.values[[0, -1]]) - fn = meta_attrs["fn"] - - logger.info(yearmonths) - logger.info([(year, month) for year, month in yearmonths]) - - return [ - dict( - prepare_func=prepare_func, - xs=xs, - ys=ys, - year=year, - month=month, - fn=fn.format(year=year, month=month, spinup=spinup_year(year, month)), - ) - for year, month in yearmonths - ] - - -weather_data_config = { - "surface_flux_hourly": dict( - api_func=api_merra2, - file_granularity="daily", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_flx_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - url_opendap="https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ), - "slv_flux_hourly": dict( - api_func=api_merra2, - file_granularity="daily_multiple", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_slv_flx_Nx.*.nc4" - ), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - url_opendap=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - "u2m", - "v2m", - "u10m", - "v10m", - "u50m", - "v50m", - ], - variables_list=[ - [ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ["u2m", "v2m", "u10m", "v10m", "u50m", "v50m"], - ], - ), - "surface_flux_monthly": dict( - api_func=api_merra2, - file_granularity="monthly", - tasks_func=tasks_monthly_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_surface_flux, - template=os.path.join(merra2_dir, "{year}/MERRA2_*.tavgM_2d_flx_Nx.*.nc4"), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXFLX.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4", - fn=os.path.join( - merra2_dir, "{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4" - ), - variables=[ - "ustar", - "z0m", - "disph", - "rhoa", - "ulml", - "vlml", - "tstar", - "hlml", - "tlml", - "pblh", - "hflux", - "eflux", - ], - ), - "surface_flux_dailymeans": dict( - api_func=api_merra2, - file_granularity="dailymeans", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_dailymeans_surface_flux, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.statD_2d_slv_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2SDNXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["hournorain", "tprecmax", "t2mmax", "t2mmean", "t2mmin"], - ), - "slv_radiation_hourly": dict( - api_func=api_merra2, - file_granularity="daily_multiple", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_slv_radiation, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_slv_rad_Nx.*.nc4" - ), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", - ], - url_opendap=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["albedo", "swgdn", "swtdn", "t2m"], - variables_list=[["t2m"], ["albedo", "swgdn", "swtdn"]], - ), - "slv_radiation_monthly": dict( - api_func=api_merra2, - file_granularity="monthly_multiple", - tasks_func=tasks_monthly_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_slv_radiation, - template=os.path.join(merra2_dir, "{year}/MERRA2_*.tavgM_2d_slv_rad_Nx.*.nc4"), - url=[ - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXSLV.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_slv_Nx.{year}{month:0>2}.nc4", - "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXRAD.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_rad_Nx.{year}{month:0>2}.nc4", - ], - fn=os.path.join( - merra2_dir, - "{year}/MERRA2_{spinup}.tavgM_2d_slv_rad_Nx.{year}{month:0>2}.nc4", - ), - variables=["albedo", "swgdn", "swtdn", "t2m"], - ), - "surface_aerosol_hourly": dict( - api_func=api_merra2, - file_granularity="daily", - tasks_func=tasks_daily_merra2, - meta_prepare_func=prepare_meta_merra2, - prepare_func=prepare_month_aerosol, - template=os.path.join( - merra2_dir, "{year}/{month:0>2}/MERRA2_*.tavg1_2d_aer_Nx.*.nc4" - ), - url="https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXAER.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", - fn=os.path.join( - merra2_dir, - "{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", - ), - variables=["bcsmass", "dusmass25", "ocsmass", "so4smass", "sssmass25"], - ), -} - -# os.path.join(merra2_dir, '{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4'), - -# list of routines in weather_data_config to download wind data -wind_files = ["surface_flux"] - -# TODO: same for solar -solar_files = [] - -## Whatever is calling this needs to be directed to the correct weather config instead -# meta_data_config = dict(prepare_func=prepare_meta_merra2, -# template=os.path.join(merra2_dir, '{year}/{month:0>2}/MERRA2_*.tavg1_2d_flx_Nx.*.nc4')) - -# Separate files for each day (coded in weather_data_config list) -# daily_files = True # needs to be specified somewhere else - -# Latitude stored south to north (ie forward, = True) or north to south -lat_direction = True - -# Spinup variable -spinup_var = True - - -def spinup_year(year, month): - if year >= 1980 and year < 1992: - spinup = "100" - elif year >= 1992 and year < 2001: - spinup = "200" - elif year >= 2001 and year < 2011: - spinup = "300" - elif year >= 2011 and year < 2020: - spinup = "400" - elif year == 2020 and month == 9: - spinup = "401" - else: - spinup = "400" - - return spinup diff --git a/src/geodata/datasets/merra2/__init__.py b/src/geodata/datasets/merra2/__init__.py new file mode 100644 index 00000000..a3c064f4 --- /dev/null +++ b/src/geodata/datasets/merra2/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import daily, hourly, monthly + +__all__ = ["hourly", "daily", "monthly"] diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py new file mode 100644 index 00000000..d5fc7639 --- /dev/null +++ b/src/geodata/datasets/merra2/_base.py @@ -0,0 +1,290 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import tempfile +from calendar import monthrange +from typing import Sequence + +import numpy as np +import requests +import xarray as xr +from tqdm.auto import tqdm + +from ...types import CoordRange, PathLike +from .._base import AtomicDataset, BaseDataset + +logger = logging.getLogger(__name__) + + +def _convert_and_subset_lons_lats_merra2( + ds: xr.Dataset, xs: CoordRange, ys: CoordRange +): + if not isinstance(xs, slice): + first, second, last = np.asarray(xs)[[0, 1, -1]] + xs = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + if not isinstance(ys, slice): + first, second, last = np.asarray(ys)[[0, 1, -1]] + ys = slice(first - 0.1 * (second - first), last + 0.1 * (second - first)) + + ds = ds.sel(y=ys) + + # Lons should go from -180. to +180. + if len(ds.coords["x"].sel(x=slice(xs.start + 360.0, xs.stop + 360.0))): + ds = xr.concat( + [ds.sel(x=slice(xs.start + 360.0, xs.stop + 360.0)), ds.sel(x=xs)], dim="x" + ) + ds = ds.assign_coords( + lon=np.where( + ds.coords["x"].values <= 180, + ds.coords["x"].values, + ds.coords["x"].values - 360.0, + ) + ) + else: + ds = ds.sel(x=xs) + + return ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"]) + + +class MERRA2BaseDataset(BaseDataset): + """MERRA2BaseDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + url_template: Sequence[str] + variables: Sequence[str] + module = "merra2" + projection = "latlong" + lat_direction = True + frequency = "daily" + + def _download_file(self, file: AtomicDataset): + assert hasattr(file, "url"), "URL is required to download the file" + + url: tuple[str] = file.url + + # Download the file + match len(file.url): + case 1: + logger.debug("Downloading %s", url[0]) + with requests.get(url[0], stream=True) as r: + r.raise_for_status() + + with ( + open(file.path, "wb") as f, + tqdm( + total=int(r.headers.get("content-length", 0)), + unit="B", + unit_scale=True, + ) as pbar, + ): + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + pbar.update(len(chunk)) + case _: + with tempfile.TemporaryDirectory() as tempdir: + for i, url in enumerate(file.url): + logger.debug("Downloading %s", url) + with requests.get(url, stream=True) as r: + r.raise_for_status() + + with ( + open(os.path.join(tempdir, str(i)), "wb") as f, + tqdm( + total=int(r.headers.get("content-length", 0)), + unit="B", + unit_scale=True, + ) as pbar, + ): + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + pbar.update(len(chunk)) + + with xr.open_mfdataset( + [os.path.join(tempdir, str(i)) for i in range(len(file.url))], + combine="by_coords", + chunks="auto", + ) as ds: + ds.to_netcdf(file.path) + logger.info("Preprocessing complete with multiple files") + + def _dataset_postprocess(self, ds: xr.Dataset, **kwargs): + """Postprocess the dataset after downloading and opening it. + + Args: + ds (xr.Dataset): The dataset to postprocess. + **kwargs: Additional keyword arguments. + + Returns: + xr.Dataset: The postprocessed dataset. + """ + + # Rename the coordinates + ds = ds.rename({"x": "lon", "y": "lat"}) + ds = ds.assign_coords(lon=ds.coords["lon"], lat=ds.coords["lat"]) + + # Only keep the variables that are needed + for var in ds.data_vars: + if var.lower() not in self.variables: + ds = ds.drop_vars(var) + + # Change all variables to lowercase + return ds.rename({var: var.lower() for var in ds.data_vars}) + + def spinup_year(self, year: int, month: int): + """Returns the spinup period for the given year and month. + See https://gmao.gsfc.nasa.gov/pubs/docs/Bosilovich785.pdf for more + information. + + Args: + year (int): The year of the dataset + month (int): The month of the dataset + + Returns: + str: The spinup period + """ + if year >= 1980 and year < 1992: + spinup = "100" + elif year >= 1992 and year < 2001: + spinup = "200" + elif year >= 2001 and year < 2011: + spinup = "300" + elif year >= 2011 and year < 2020: + spinup = "400" + elif year == 2020 and month == 9: + spinup = "401" + else: + spinup = "400" + + return spinup + + def _daily_catalog(self): + if not self.url_template: + raise NotImplementedError("url_template is not defined for this dataset") + + catalog = super()._daily_catalog() + + for file in catalog: + file.spinup = self.spinup_year(file.year, file.month) + file.url = [t.format(**vars(file)) for t in self.url_template] + + return catalog + + def _monthly_catalog(self): + if not self.url_template: + raise NotImplementedError("url_template is not defined for this dataset") + + catalog = super()._monthly_catalog() + + for file in catalog: + file.spinup = self.spinup_year(file.year, file.month) + file.url = [t.format(**vars(file)) for t in self.url_template] + + return catalog + + @classmethod + def meta_prepare_func( + cls, xs: CoordRange, ys: CoordRange, year: int, month: int, **params + ): + with xr.open_mfdataset(cls._get_files(year, month), combine="by_coords") as ds: + ds = ds.coords.to_dataset() + ds = _convert_and_subset_lons_lats_merra2(ds, xs, ys) + meta = ds.load() + + return meta + + @classmethod + def tasks_func( + cls, + xs: CoordRange, + ys: CoordRange, + yearmonths: xr.DataArray, + **meta_attrs, + ): + if not isinstance(xs, slice): + xs = slice(*xs.values[[0, -1]]) + if not isinstance(ys, slice): + ys = slice(*ys.values[[0, -1]]) + fn = meta_attrs["fn"] + + match cls.frequency: + case "daily": + logger.info(yearmonths) + logger.info( + [ + (year, month, day) + for year, month in yearmonths + for day in range(1, monthrange(year, month)[1] + 1, 1) + ] + ) + + return [ + dict( + prepare_func=cls.prepare_func, + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format( + year=year, + month=month, + day=day, + spinup=cls.spinup_year(year, month), + ), + ) + for year, month in yearmonths + for day in range(1, monthrange(year, month)[1] + 1, 1) + ] + case "monthly": + return [ + dict( + xs=xs, + ys=ys, + year=year, + month=month, + fn=fn.format(year=year, month=month), + ) + for year, month in yearmonths + ] + case _: + raise NotImplementedError("Frequency not supported") + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month.""" + + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn) as ds: + logger.info("Opening %s", fn) + ds = _convert_and_subset_lons_lats_merra2(ds, xs, ys) + ds = ds.rename({"x": "lon", "y": "lat"}) + ds = ds.assign_coords(lon=ds.coords["lon"], lat=ds.coords["lat"]) + + yield (year, month), ds diff --git a/src/geodata/datasets/merra2/daily/__init__.py b/src/geodata/datasets/merra2/daily/__init__.py new file mode 100644 index 00000000..7bf31a96 --- /dev/null +++ b/src/geodata/datasets/merra2/daily/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .surface_flux import MERRA2SurfaceFluxDailyDataset + +__all__ = ["MERRA2SurfaceFluxDailyDataset"] diff --git a/src/geodata/datasets/merra2/daily/surface_flux.py b/src/geodata/datasets/merra2/daily/surface_flux.py new file mode 100644 index 00000000..4cd4f577 --- /dev/null +++ b/src/geodata/datasets/merra2/daily/surface_flux.py @@ -0,0 +1,31 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxDailyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_daily" + + variables = ["hournorain", "tprecmax", "t2mmax", "t2mmean", "t2mmin"] + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2SDNXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.statD_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/__init__.py b/src/geodata/datasets/merra2/hourly/__init__.py new file mode 100644 index 00000000..29db1c06 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .slv_flux import MERRA2SLVFluxHourlyDataset +from .slv_radiation import MERRA2SLVRadiationHourlyDataset +from .surface_aerosol import MERRA2SurfaceAerosolHourlyDataset +from .surface_flux import MERRA2SurfaceFluxHourlyDataset + +__all__ = [ + "MERRA2SurfaceFluxHourlyDataset", + "MERRA2SLVFluxHourlyDataset", + "MERRA2SLVRadiationHourlyDataset", + "MERRA2SurfaceAerosolHourlyDataset", +] diff --git a/src/geodata/datasets/merra2/hourly/slv_flux.py b/src/geodata/datasets/merra2/hourly/slv_flux.py new file mode 100644 index 00000000..3edfb8c6 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/slv_flux.py @@ -0,0 +1,52 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVFluxHourlyDataset(MERRA2BaseDataset): + """MERRA2SLVFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_flux_hourly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + "u2m", + "v2m", + "u10m", + "v10m", + "u50m", + "v50m", + ] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/slv_radiation.py b/src/geodata/datasets/merra2/hourly/slv_radiation.py new file mode 100644 index 00000000..74c61739 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/slv_radiation.py @@ -0,0 +1,33 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVRadiationHourlyDataset(MERRA2BaseDataset): + """MERRA2SLVRadiationHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_radiation_hourly" + + variables = ["albedo", "swgdn", "swtdn", "t2m"] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXSLV.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_slv_Nx.{year}{month:0>2}{day:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXRAD.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_rad_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/surface_aerosol.py b/src/geodata/datasets/merra2/hourly/surface_aerosol.py new file mode 100644 index 00000000..c1ade9f0 --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/surface_aerosol.py @@ -0,0 +1,32 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceAerosolHourlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceAerosolHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_aerosol_hourly" + + variables = ["bcsmass", "dusmass25", "ocsmass", "so4smass", "sssmass25"] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXAER.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_aer_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/hourly/surface_flux.py b/src/geodata/datasets/merra2/hourly/surface_flux.py new file mode 100644 index 00000000..523b470b --- /dev/null +++ b/src/geodata/datasets/merra2/hourly/surface_flux.py @@ -0,0 +1,44 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxHourlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxHourlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_hourly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + ] + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/M2T1NXFLX.5.12.4/{year}/{month:0>2}/MERRA2_{spinup}.tavg1_2d_flx_Nx.{year}{month:0>2}{day:0>2}.nc4", + ) diff --git a/src/geodata/datasets/merra2/monthly/__init__.py b/src/geodata/datasets/merra2/monthly/__init__.py new file mode 100644 index 00000000..4fc2414b --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .slv_radiation import MERRA2SLVRadiationMonthlyDataset +from .surface_flux import MERRA2SurfaceFluxMonthlyDataset + +__all__ = ["MERRA2SLVRadiationMonthlyDataset", "MERRA2SurfaceFluxMonthlyDataset"] diff --git a/src/geodata/datasets/merra2/monthly/slv_radiation.py b/src/geodata/datasets/merra2/monthly/slv_radiation.py new file mode 100644 index 00000000..201d1bdb --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/slv_radiation.py @@ -0,0 +1,33 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SLVRadiationMonthlyDataset(MERRA2BaseDataset): + """MERRA2SLVRadiationMonthlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "slv_radiation_monthly" + frequency = "monthly" + variables = ["albedo", "swgdn", "swtdn", "t2m"] + + url_template = [ + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXSLV.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_slv_Nx.{year}{month:0>2}.nc4", + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXRAD.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_rad_Nx.{year}{month:0>2}.nc4", + ] diff --git a/src/geodata/datasets/merra2/monthly/surface_flux.py b/src/geodata/datasets/merra2/monthly/surface_flux.py new file mode 100644 index 00000000..f3f71300 --- /dev/null +++ b/src/geodata/datasets/merra2/monthly/surface_flux.py @@ -0,0 +1,45 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from .._base import MERRA2BaseDataset + + +class MERRA2SurfaceFluxMonthlyDataset(MERRA2BaseDataset): + """MERRA2SurfaceFluxMonthlyDataset is a class that encaps a dataset from the MERRA2 reanalysis + dataset. It provides a streamlined workflow for downloading, preprocessing, + and storing of these datasets. + """ + + weather_config = "surface_flux_monthly" + + variables = [ + "ustar", + "z0m", + "disph", + "rhoa", + "ulml", + "vlml", + "tstar", + "hlml", + "tlml", + "pblh", + "hflux", + "eflux", + ] + + url_template = ( + "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2_MONTHLY/M2TMNXFLX.5.12.4/{year}/MERRA2_{spinup}.tavgM_2d_flx_Nx.{year}{month:0>2}.nc4", + ) diff --git a/src/geodata/logging.py b/src/geodata/logging.py index a3bab789..78018a85 100644 --- a/src/geodata/logging.py +++ b/src/geodata/logging.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import contextlib import logging as _logging - color2num = { "gray": 30, "red": 31, @@ -54,6 +54,53 @@ def colorize( return f"\x1b[{attrs}m{string}\x1b[0m" +class StdoutToLoggerRedirect: + """A class to redirect stdout to a logger. + + Args: + logger: The logger to redirect stdout to. If not provided, a new logger + will be created. + level: The logging level to use when redirecting stdout. + Default is logging.INFO. + + Example: + >>> import logging + >>> with contextlib.redirect_stdout(StdoutToLoggerRedirect()): + ... print("Hello, world!") + """ + + def __init__( + self, logger: _logging.Logger | None = None, level: int = _logging.INFO + ): + self.logger = logger or _logging.getLogger(__name__) + self.level = level + + def write(self, msg: str): + if msg and not msg.isspace(): + self.logger.log(self.level, msg) + + def flush(self): + pass + + +@contextlib.contextmanager +def redirect_stdout_to_logger(logger=None, level=_logging.INFO): + """Context manager to redirect stdout to a logger. + + Args: + logger: The logger to redirect stdout to. If not provided, a new logger + will be created. + level: The logging level to use. Default is logging.INFO. + + Example: + >>> import logging + >>> with redirect_stdout_to_logger(): + ... print("Hello, world!") + """ + with contextlib.redirect_stdout(StdoutToLoggerRedirect(logger, level)): + yield + + class CustomFormatter(_logging.Formatter): # https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output @@ -71,7 +118,11 @@ def format(self, record): logger = _logging.getLogger("geodata") -logger.setLevel(_logging.INFO) + +# Only set the level if it hasn't been set yet +if logger.level == _logging.NOTSET: + logger.setLevel(_logging.INFO) + logger.propagate = False if not logger.hasHandlers(): ch = _logging.StreamHandler() @@ -79,3 +130,5 @@ def format(self, record): CustomFormatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ) logger.addHandler(ch) + +__all__ = ["logger", "redirect_stdout_to_logger"] diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index 3174c7e3..a8a89bd7 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -15,21 +15,40 @@ import abc -import hashlib -import json +import importlib.util +import os import shutil -from pathlib import Path -from typing import Optional, Union +from typing import Optional import xarray as xr - -import geodata +from tqdm.auto import tqdm from ..config import model_dir -from ..cutout import Cutout -from ..dataset import Dataset +from ..datasets._base import BaseDataset from ..logging import logger -from ..utils import NpEncoder +from .results import DailyModelResult, MonthlyModelResult, ResultType + +if importlib.util.find_spec("h5netcdf") is not None: + XR_PARALLEL = True + XR_ENGINE = "h5netcdf" +else: + XR_PARALLEL = False + XR_ENGINE = None + logger.warning( + "h5netcdf is not installed. Parallel reading of netCDF files will be disabled. " + "This could have some performance implications." + ) + +# Parse the MAX_WORKERS environment variable if present +MAX_WORKERS = os.getenv("MAX_WORKERS") +if MAX_WORKERS is not None: + try: + max_workers = int(MAX_WORKERS) + except ValueError: + logger.warning( + "MAX_WORKERS environment variable is not an integer. Using default value." + ) + MAX_WORKERS = None class BaseModel(abc.ABC): @@ -37,219 +56,149 @@ class BaseModel(abc.ABC): Args: name (str): The name of the model. - source (Union[geodata.Dataset, geodata.Cutout, xr.Dataset]): The source of the model. Can be a Dataset, Cutout or xarray.Dataset. + source (BaseDataset): The source of the model. interpolate (bool, optional): Interpolate the source to the same grid as the target. Defaults to False. + quick_check (bool, optional): Quick check for the model. Defaults to False. If True, the model parameters will be checked for presence, but not the integrity. **kwargs: Additional keyword arguments to pass to the model. """ SUPPORTED_WEATHER_DATA_CONFIGS: tuple[str] - metadata_keys: set[str] = { - "name", - "module", - "from_dataset", - "years", - "months", - "files_orig", - "files_prepared", - "weather_data_config", - } - - def __init__(self, source: Union[geodata.Dataset, geodata.Cutout], **kwargs): - if source.config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: - raise ValueError( - f"Weather data config {source.config} is not supported by this model." - ) - if not source.prepared: + def __init__(self, source: BaseDataset, **kwargs): + if source.weather_config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: raise ValueError( - "The source Dataset/Cutout for this model is not prepared." + f"Weather data config {source.weather_config} is not supported by this model." ) + if not source.downloaded: + raise ValueError("The source Dataset for this model is not prepared.") + self.source = source + self.quick_check = kwargs.get("quick_check", False) self._extra_kwargs = kwargs - self._corrupt_metadata = False self._prepared = False - if isinstance(source, geodata.Dataset): - self._ref_path = model_dir.parent / self.source.module - self._path = ( - model_dir - / self.type - / self.__class__.__name__ - / "datasets" - / self.source.module - ) - else: - self._ref_path = model_dir.parent / "cutouts" - self._path = ( - model_dir - / self.type - / self.__class__.__name__ - / "cutouts" - / self.source.name - ) - - if (meta_path := self._path / "meta.json").exists(): - try: - with open(meta_path, encoding="utf_8") as f: - self.metadata = json.load(f) - - # NOTE: Double-check if we have an updated dataset/cutout - # If we do, we need to re-prepare the model - if isinstance(source, geodata.Dataset): - _source_files = self.extract_dataset_metadata(source).get( - "files_orig", {} - ) - else: - _source_files = self.extract_cutout_metadata(source).get( - "files_orig", {} - ) - - if set(_source_files.keys()) != set( - self.metadata.get("files_orig", {}).keys() - ): - logger.warning( - "Metadata file %s is outdated. Model will be re-prepared.", - meta_path, - ) - self._corrupt_metadata = True - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) - - except json.JSONDecodeError: - logger.warning( - "Metadata file %s is corrupted. Model will be re-prepared.", - meta_path, - ) - self._corrupt_metadata = True - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) - else: - # NOTE: We only store metadata in transient fashion until preparation is done - if isinstance(source, geodata.Dataset): - self.metadata = self.extract_dataset_metadata(source) - else: - self.metadata = self.extract_cutout_metadata(source) + self._ref_path = model_dir.parent / self.source.module + self._results: dict[int, dict[int, ResultType]] = self._prepare_results() def __repr__(self): return f"Model(source={self.source}, type={self.type})" @property - def name(self) -> str: - """Name of the model.""" - return f"{self.metadata['name']}_{self.type}" - - @property - def module(self) -> str: - """Module of the model.""" - return self.metadata["module"] + def frequency(self) -> str: + """Frequency of the model.""" + return self.source.frequency @property @abc.abstractmethod def type(self) -> str: """Type of the model.""" + def _prepare_results(self) -> dict[int, dict[int, ResultType]]: + """Prepare the results of the model. + + Returns: + dict: Dictionary with the results of the model. + """ + + years = list(range(self.source.years.start, self.source.years.stop + 1)) + months = list(range(self.source.months.start, self.source.months.stop + 1)) + + results: dict[int, dict[int, ResultType]] = {} + for year in years: + results[year] = {} + for month in months: + match self.frequency: + case "daily" | "hourly": + results[year][month] = DailyModelResult.from_year_month( + self, year, month + ) + case "monthly": + results[year][month] = MonthlyModelResult.from_year_month( + self, year, month + ) + results[year][month].path.mkdir(parents=True, exist_ok=True) + + return results + + @property + def results(self): + """Get the results of the model. + + Returns: + dict: Dictionary with the results of the model. + """ + return self._results + + def get_result_year_month(self, years: slice, months: slice) -> list[ResultType]: + """Get the result of the model for a given year and month range. + + Args: + years (slice): Year range. + months (slice): Month range. + Returns: + list: List of DailyModelResult objects. + """ + year_d = [self._results[y] for y in range(years.start, years.stop + 1)] + return [y[m] for y in year_d for m in range(months.start, months.stop + 1)] + + @property + def flattened_results(self) -> list[ResultType]: + """Flatten the results of the model. + + Returns: + list: List of ModelResult objects. + """ + + return [ + self._results[year][month] + for year in self._results + for month in self._results[year] + ] + def estimate( self, - height: int, - years: slice, + years: Optional[slice] = None, months: Optional[slice] = None, xs: Optional[slice] = None, ys: Optional[slice] = None, - use_real_data: bool = False, + **kwargs, ) -> xr.DataArray: """Estimate the wind speed at given coordinates. Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. + years (slice, optional): Years. months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. + xs (slice, optional): X coordinates. If None, all x coordinates in source are estimated. + ys (slice, optional): Y coordinates. If None, all y coordinates in source are estimated. + **kwargs: Additional keyword arguments to pass to the model. Returns: xr.DataArray: Dataset with wind speed. """ - - if self.from_dataset: - return self._estimate_dataset( - height=height, - years=years, - months=months, - xs=xs, - ys=ys, - use_real_data=use_real_data, + if not self.prepared: + raise RuntimeError( + "The model is not prepared. Please prepare the model first." ) - return self._estimate_cutout( - height=height, - years=years, - months=months, - xs=xs, - ys=ys, - use_real_data=use_real_data, - ) - - def extract_dataset_metadata(self, dataset: Dataset) -> dict: - if not dataset.prepared: - raise ValueError("The source dataset for this model is not prepared.") - - logger.info("Using dataset %s", dataset.module) - - metadata = {} - - metadata["name"] = metadata["module"] = dataset.module - metadata["from_dataset"] = True - - if isinstance(dataset.years, slice): - metadata["years"] = dataset.years.start, dataset.years.stop - if isinstance(dataset.months, slice): - metadata["months"] = dataset.months.start, dataset.months.stop - - metadata["weather_data_config"] = dataset.config - - # NOTE: file paths for estimation parameters will be added later in the prepare step - metadata["files_prepared"] = {} - metadata["files_orig"] = {} - for c, fp in dataset.downloadedFiles: - if c == metadata["weather_data_config"]: - with open(fp, "rb") as f: - metadata["files_orig"][ - str(Path(fp).relative_to(self._ref_path)) - ] = hashlib.sha256(f.read()).hexdigest() - - return metadata - - def extract_cutout_metadata(self, cutout: Cutout) -> dict: - logger.info("Using cutout %s", cutout.name) - - metadata = {} - - metadata["name"] = cutout.name - metadata["module"] = cutout.meta.attrs["module"] - metadata["from_dataset"] = False - metadata["weather_data_config"] = cutout.config + if years is None and months is None: + results = self.flattened_results + elif months is None: + results = self.get_result_year_month(years, slice(1, 13)) + else: + results = self.get_result_year_month(years, months) - if isinstance(cutout.years, slice): - metadata["years"] = cutout.years.start, cutout.years.stop - if isinstance(cutout.months, slice): - metadata["months"] = cutout.months.start, cutout.months.stop + files = sum([result.files for result in results], []) + params = xr.open_mfdataset(files, engine=XR_ENGINE, parallel=XR_PARALLEL) - metadata["files_prepared"] = {} - metadata["files_orig"] = {} - for yearmonth in cutout.coords["year-month"].to_index(): - fp = cutout.datasetfn(yearmonth) - with open(fp, "rb") as f: - metadata["files_orig"][str(Path(fp).relative_to(self._ref_path))] = ( - hashlib.sha256(f.read()).hexdigest() - ) + if xs is not None: + params = params.sel(x=xs) + if ys is not None: + params = params.sel(y=ys) - return metadata + output = self._estimate_dataset(params, **kwargs) + params.close() + return output @property def prepared(self) -> bool: @@ -264,161 +213,56 @@ def prepared(self) -> bool: return self._prepared def _check_prepared(self) -> bool: - assert self._path is not None, "The model saving path has not been set yet." - - nc4_path: Path = self._path / "nc4" - meta_path: Path = self._path / "meta.json" - - if not nc4_path.exists() or not meta_path.exists() or self._corrupt_metadata: - return False - with open(meta_path, encoding="utf-8") as f: - metadata_loaded = json.load(f) - if set(metadata_loaded.keys()) != self.metadata_keys: - return False - - if len(self.metadata["files_orig"]) != len(self.metadata["files_prepared"]): - return False - - nc4_rel_path = nc4_path.relative_to(self._path) - for fp in self.metadata["files_orig"]: - fp_prepared = str(nc4_rel_path / Path(fp).with_suffix(".params.nc4")) - - if ( - fp not in self.metadata["files_orig"] - or fp_prepared not in self.metadata["files_prepared"] - ): - return False - - with open(self._ref_path / fp, "rb") as f: - if ( - self.metadata["files_orig"][fp] - != hashlib.sha256(f.read()).hexdigest() - ): - logger.warning( - "File %s in source dataset has been modified since model creation. Model is not prepared!", - fp, - ) + for year in self._results: + for month in self._results[year]: + if not self._results[year][month].prepared: return False - - with open(self._path / fp_prepared, "rb") as f: - if ( - self.metadata["files_prepared"][fp_prepared] - != hashlib.sha256(f.read()).hexdigest() - ): - logger.warning( - "Parameter file %s in model has been modified since model creation. Model is not prepared!", - fp_prepared, - ) - return False - return True def prepare(self, force: bool = False): """Prepare the model. Args: - force (bool, optional): Force re-prepare the model. Defaults to False.""" + force (bool, optional): Force re-prepare the model. Defaults to False. + """ - self._prepared = False # NOTE: force re-checking preparedness here if self.prepared and not force: logger.info("The model is already prepared.") return - logger.info("Model not present in model directory, creating.") - shutil.rmtree(self._path, ignore_errors=True) - (self._path / "nc4").mkdir(exist_ok=True, parents=True) - - self.metadata["files_prepared"] = {} - for fp in ( - self._prepare_dataset() if self.from_dataset else self._prepare_cutout() - ): - with open(self._path / fp, "rb") as f: - self.metadata["files_prepared"][fp] = hashlib.sha256( - f.read() - ).hexdigest() - - with open(self._path / "meta.json", "w", encoding="utf-8") as f: - json.dump(self.metadata, f, indent=4, cls=NpEncoder) + for result in tqdm(self.flattened_results): + if not result.prepared or force: + shutil.rmtree(result.path, ignore_errors=True) + result.path.mkdir(parents=True, exist_ok=True) - logger.info("Finished preparing model.") - - @abc.abstractmethod - def _prepare_dataset(self) -> list: - """Prepare the model from a dataset. - - Returns: - list: List of files. - """ - - @abc.abstractmethod - def _prepare_cutout(self) -> list: - """Prepare the model from a cutout. - - Returns: - list: List of files. - """ + with xr.open_mfdataset( + result.ref_files, engine=XR_ENGINE, parallel=XR_PARALLEL + ) as ds: + prepared_ds = self._prepare_dataset(ds) + result.register(prepared_ds) - @property - def from_dataset(self) -> bool: - """Check if the model is from a dataset.""" - if "from_dataset" not in self.metadata: - return False - return self.metadata["from_dataset"] + result.dump() + logger.info("Model prepared successfully.") @abc.abstractmethod - def _estimate_dataset( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: - """Estimate the wind speed from a dataset. + def _prepare_dataset(self, source: xr.Dataset) -> xr.Dataset: + """Prepare the parameters of a specific source dataset file. Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. - months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. + source (xr.Dataset): Source dataset. Returns: - xr.DataArray: Dataset with wind speed. + xr.Dataset: Prepared parameter dataset. """ @abc.abstractmethod - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: - """Estimate the wind speed from a cutout. + def _estimate_dataset(self, params: xr.Dataset, **kwargs) -> xr.DataArray: + """Estimate the wind speed from a dataset. Args: - height (int): Height of the wind speed, need to be greater than 0. - years (slice): Years. - months (slice, optional): Months. If None, all months are estimated. - xs (slice): X coordinates. If None, all x coordinates in source are estimated. - ys (slice): Y coordinates. If None, all y coordinates in source are estimated. - use_real_data (bool, optional): If available, use real data for estimation. Defaults to False. + params (xr.Dataset): Parameters of the model. + **kwargs: Additional keyword arguments to pass to the model. Returns: - xr.DataArray: Dataset with wind speed. + xr.DataArray: Result after modeling. """ - - @property - def files(self): - if "files_prepared" not in self.metadata or "files_orig" not in self.metadata: - return [] - - files_prepared = [self._path / p for p in self.metadata["files_prepared"]] - files_orig = [self._ref_path / p for p in self.metadata["files_orig"]] - - return files_prepared + files_orig diff --git a/src/geodata/model/results/__init__.py b/src/geodata/model/results/__init__.py new file mode 100644 index 00000000..0f3bc4f7 --- /dev/null +++ b/src/geodata/model/results/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""An interface for model computation results.""" + +from .daily import DailyModelResult +from .monthly import MonthlyModelResult + +ResultType = DailyModelResult | MonthlyModelResult + +__all__ = ["DailyModelResult", "MonthlyModelResult", "ResultType"] diff --git a/src/geodata/model/results/_base.py b/src/geodata/model/results/_base.py new file mode 100644 index 00000000..2430f01d --- /dev/null +++ b/src/geodata/model/results/_base.py @@ -0,0 +1,257 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from __future__ import annotations + +import abc +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +import xarray as xr + +from geodata.config import model_dir + +if TYPE_CHECKING: + from typing import Self + + from .._base import BaseModel + +logger = logging.getLogger(__name__) + + +@dataclass +class BaseModelResult(abc.ABC): + """Model result class. This class is used to store the result of a model. + All results, regardless of the actual storage format, are grouped in monthly + instances. + + Args: + year (int): Year of the model. + month (int): Month of the model. + model (BaseModel): Model object that represent the downloaded raw data. + """ + + year: int + month: int + model: BaseModel + + _hashes: dict[str, str] = field(default_factory=dict) + _prepared: bool = False + _frequency: str = field(init=False) + + def __post_init__(self): + if hasattr(self, "_frequency") and self._frequency != self.model.frequency: + raise ValueError( + f"Model frequency {self.model.frequency} does not match {self._frequency}." + ) + self._frequency = self.model.frequency + + @property + def frequency(self) -> str: + """Frequency of the model.""" + return self.model.frequency + + @property + def module(self) -> str: + """Module of the model.""" + return self.model.source.module + + @property + def ref_path(self) -> Path: + """Path to the reference dataset.""" + return ( + self.model._ref_path + / self.model.source.weather_config + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def files(self) -> list[Path]: + """List of files in the model dataset. This could potentially include any + files that are not prepared yet.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Model files for {self.year:04d}-{self.month:02d} not found." + ) + + files = [] + for f in atomic_files: + p = f.path.relative_to(self.ref_path) + files.append(self.path / p) + return files + + @property + def ref_files(self) -> list[Path]: + """List of files in the reference dataset.""" + + atomic_files = self.model.source.get_monthly_catalog(self.year, self.month) + if atomic_files is None: + raise ValueError( + f"Reference files for {self.year:04d}-{self.month:02d} not found." + ) + + return [f.path for f in atomic_files if f.path.exists()] + + @property + def ref_params(self) -> dict[Path, Path]: + """Reference parameters of the model.""" + + ref_params = {} + for file in self.ref_files: + if file.name.endswith(".params.nc"): + ref_params[file] = self.path / file.name + else: + ref_params[file] = self.path / f"{file.stem}.params.nc" + return ref_params + + @property + def path(self) -> Path: + """Path to the model dataset.""" + return ( + model_dir + / self.module + / self.model.__class__.__name__ + / f"{self.year:04d}" + / f"{self.month:02d}" + ) + + @property + def prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + if not self._prepared: + self._prepared = self._check_prepared() + return self._prepared + + @abc.abstractmethod + def _check_prepared(self) -> bool: + """Check if the model is prepared. + + Returns: + bool: True if prepared. + """ + + @abc.abstractmethod + def register(self, dataset: xr.Dataset): + """Register the model result with the dataset. + + Args: + dataset (xr.Dataset): Dataset to register. + """ + + @classmethod + def from_year_month(cls, model: BaseModel, year: int, month: int) -> Self: + """Create an ModelResult from a year and month. + + Args: + model (BaseModel): BaseModel object. + year (int): Year of the model. + month (int): Month of the model. + + Returns: + AtomicModel: AtomicModel object. + """ + + if not (1 <= month <= 12): + raise ValueError(f"Month {month} is not valid. Must be between 1 and 12.") + if not (2000 <= year <= 2100): + raise ValueError( + f"Year {year} is not valid. Must be between 2000 and 2100." + ) + + path = ( + model_dir + / model.source.module + / model.__class__.__name__ + / f"{year:04d}" + / f"{month:02d}" + / "meta.json" + ) + + if not path.exists(): + return cls(model=model, year=year, month=month) + + with open(path, "r") as f: + data = json.load(f) + + if data["year"] != year or data["month"] != month: + raise ValueError( + f"Model year {data['year']} and month {data['month']} do not match {year} and {month}." + ) + + return cls.from_dict(data, model) + + def __repr__(self): + return f"AtomicModel(year={self.year}, month={self.month}, ref_path={self.ref_path}, path={self.path} {len(self.files)} / {len(self.ref_files)})" + + @classmethod + def from_dict(cls, data: dict, model: BaseModel) -> Self: + """Create an AtomicModel from a dictionary. + + Args: + data (dict): Dictionary with the model data. + + Returns: + AtomicModel: AtomicModel object. + """ + + inst = cls(year=data["year"], month=data["month"], model=model) + + inst._hashes = data.get("hashes", {}) + inst.prepared + return inst + + def to_dict(self) -> dict: + """Convert the AtomicModel to a dictionary. + + Returns: + dict: Dictionary with the model data. + """ + + return { + "year": self.year, + "month": self.month, + "ref_path": str(self.ref_path), + "path": str(self.path), + "hashes": self._hashes, + "prepared": self.prepared, + } + + def dump(self): + """Dump the model result to a file. + + Returns: + dict: Dictionary with the model data. + """ + info = self.to_dict() + with open(self.path / "meta.json", "w") as f: + json.dump(info, f, indent=4) + + # We dump again to check for meta preparedness + info = self.to_dict() + with open(self.path / "meta.json", "w") as f: + json.dump(info, f, indent=4) + + logger.info("Model result dumped to %s", self.path / "meta.json") diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py new file mode 100644 index 00000000..1a60d0cf --- /dev/null +++ b/src/geodata/model/results/daily.py @@ -0,0 +1,143 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import hashlib +import logging +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path + +import xarray as xr +from tqdm.auto import tqdm + +from geodata.utils import check_hash + +from ._base import BaseModelResult + +logger = logging.getLogger(__name__) + + +@dataclass +class DailyModelResult(BaseModelResult): + """Class for daily model results.""" + + def _check_prepared(self): + assert self.path is not None, "The model saving path has not been set yet." + + if not (self.path / "meta.json").exists(): + logger.warning( + "Model %s-%s does not have metadata. Please prepare the model first.", + self.year, + self.month, + ) + return False + + if self.model.quick_check: + # If the quick check is enabled, we only need to check the file hashes + # and not the actual data. + logger.debug( + "Quick check is enabled. Only checking file hashes for model %s-%s.", + self.year, + self.month, + ) + + for file in self.files: + if not file.exists(): + logger.warning( + "File %s in model does not exist. Model is not prepared!", + str(file), + ) + return False + + return True + from .._base import MAX_WORKERS + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + files = [(f, self._hashes.get(f.name)) for f in self.files] + results = list( + tqdm( + executor.map(lambda t: check_hash(*t), files), + total=len(files), + unit="file", + dynamic_ncols=True, + desc=f"Model Files Integrity Check {self.year:04d}-{self.month:02d}", + ) + ) + + for file, (is_valid, hash_value) in zip(files, results): + if not is_valid: + logger.warning( + "File %s in model has been modified since model creation. Model is not prepared!", + str(file[0]), + ) + return False + + return True + + def register(self, dataset: xr.Dataset): + """Register the model result with the dataset. This file should be a file + covering the entire model period. + + Args: + dataset (xr.Dataset): The dataset to register with. + """ + + time = dataset.get("valid_time") + + start = time.min() + end = time.max() + + if start.dt.month != self.month or end.dt.month != self.month: + raise ValueError( + f"Dataset start month {start.dt.month} does not match model month {self.month}." + ) + + if start.dt.year != self.year or end.dt.year != self.year: + raise ValueError( + f"Dataset start year {start.dt.year} does not match model year {self.year}." + ) + + days, datasets = zip(*dataset.groupby("valid_time.day")) + paths = [self.path / f"{day:02d}.nc" for day in days] + + logger.debug("Saving model results to %s", self.path) + + from .._base import XR_ENGINE + + xr.save_mfdataset(datasets, paths, engine=XR_ENGINE) + + # Write the hash file for integrity checking + with ThreadPoolExecutor() as executor: + + def compute_hash(path: Path): + sha256 = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + return path.name, sha256.hexdigest() + + results = list( + tqdm( + executor.map(compute_hash, paths), + total=len(paths), + unit="file", + dynamic_ncols=True, + desc="Computing File Hashes", + ) + ) + + for k, v in results: + self._hashes[k] = v diff --git a/src/geodata/model/results/monthly.py b/src/geodata/model/results/monthly.py new file mode 100644 index 00000000..b8648ffa --- /dev/null +++ b/src/geodata/model/results/monthly.py @@ -0,0 +1,51 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import hashlib +import logging +from dataclasses import dataclass + +import xarray as xr + +from geodata.utils import check_hash + +from ._base import BaseModelResult + +logger = logging.getLogger(__name__) + + +@dataclass +class MonthlyModelResult(BaseModelResult): + """Class for monthly model results.""" + + def _check_prepared(self): + assert self.path is not None, "The model saving path has not been set yet." + + if not (self.path / "meta.json").exists(): + logger.warning( + "Model %s-%s does not have metadata. Please prepare the model first.", + self.year, + self.month, + ) + return False + + return check_hash(self.path / f"{self.month:02d}.params.nc")[0] + + def register(self, dataset: xr.Dataset): + dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc") + with open(self.path / f"{self.month:02d}.params.nc", "rb") as f: + self._hashes[f"{self.month:02d}.params.nc"] = hashlib.sha256( + f.read() + ).hexdigest() diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index 96e46622..244f81e3 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -39,15 +39,13 @@ >>> model.estimate(xs=slice(1, 2), ys=slice(1, 2), years=slice(2010, 2010), months=slice(1, 2)) """ -from pathlib import Path -from typing import Callable - import xarray as xr -from tqdm.auto import tqdm -from ...logging import logger +from ...resource import get_windturbineconfig from .._base import BaseModel +from scipy.interpolate import interp1d + HEIGHTS = {"u50m": 50, "u10m": 10, "u2m": 2} @@ -61,62 +59,92 @@ class WindBaseModel(BaseModel): """ type: str = "wind" - _prepare_fn: Callable[[xr.Dataset], xr.Dataset] - - def _prepare_dataset(self) -> list[tuple[str, Path]]: - """Prepare the model from a dataset.""" - - logger.info("Preparing the model from dataset.") - - prepared_files = [] - for file_path in tqdm(self.metadata["files_orig"], dynamic_ncols=True): - orig_ds_path: Path = self._ref_path / file_path - ds = xr.open_dataset(orig_ds_path, chunks="auto") - try: - ds = self._prepare_fn(ds) - except SystemError: - logger.warning( - "Could not compute wind speed of %s, possibly due to corrupt file.", - orig_ds_path.name, - ) - continue - - ds_path: Path = ( - self._path / "nc4" / Path(file_path).with_suffix(".params.nc4") - ) - ds_path.parent.mkdir(parents=True, exist_ok=True) - ds.to_netcdf(ds_path) - - prepared_files.append(str(ds_path.relative_to(self._path))) - - return prepared_files - def _prepare_cutout(self) -> list[tuple[str, Path]]: - """Prepare the model from a cutout.""" - - logger.info("Preparing the model from cutout.") - prepared_files = [] - - for yearmonth in tqdm(self.source.coords["year-month"].to_index()): - orig_ds_path = Path(self.source.datasetfn(yearmonth)) - - ds = xr.open_dataset(orig_ds_path) - try: - ds = self._prepare_fn(ds) - except SystemError: - logger.warning( - "Could not compute wind speed of %s, possibly due to corrupt file.", - orig_ds_path.name, - ) - continue - - ds_path = orig_ds_path.relative_to(self._ref_path).with_suffix( - ".params.nc4" + def estimate( + self, + years: slice | None = None, + months: slice | None = None, + xs: slice | None = None, + ys: slice | None = None, + **kwargs, + ) -> xr.DataArray: + """Estimate wind speed or CF at the given locations and times. If a turbine is + specified, the CF is calculated based on the wind speed and the turbine's power curve. + Otherwise, pass in the `height` keyword argument to estimate wind speed at a specific height. + + Args: + years (slice, optional): Years. + months (slice, optional): Months. If None, all months are estimated. + xs (slice, optional): X coordinates. If None, all x coordinates in source are estimated. + ys (slice, optional): Y coordinates. If None, all y coordinates in source are estimated. + **kwargs: Additional keyword arguments to pass to the model. + - `turbine` (str): Name of the wind turbine to estimate power output. + - `height` (int): Height at which to estimate wind speed. If not specified, the model will use the default height. + + Raises: + ValueError: If neither 'turbine' nor 'height' is specified in kwargs. + + Returns: + xr.DataArray: Estimated wind speed. + """ + + if "turbine" in kwargs: + return self._estimate_power( + years=years, months=months, xs=xs, ys=ys, **kwargs ) - ds_path = self._path / "nc4" / ds_path - ds_path.parent.mkdir(parents=True, exist_ok=True) - ds.to_netcdf(ds_path) - prepared_files.append(str(ds_path.relative_to(self._path))) + if "height" not in kwargs: + raise ValueError( + "Either 'turbine' or 'height' must be specified to estimate wind speed." + ) - return prepared_files + return super().estimate(years, months, xs, ys, **kwargs) + + def _estimate_power( + self, + turbine: str, + xs: slice | None = None, + ys: slice | None = None, + years: slice | None = None, + months: slice | None = None, + ) -> None: + """Estimate wind speed at the given locations and times. + + Args: + turbine (str): Turbine name. + xs (slice, optional): X slice. Defaults to None. + ys (slice, optional): Y slice. Defaults to None. + years (slice, optional): Year slice. Defaults to None. + months (slice, optional): Month slice. Defaults to None. + + Returns: + xr.DataArray: Estimated wind speed. + """ + + # Get the wind turbine configuration + try: + turbineconf = get_windturbineconfig(turbine) + except FileNotFoundError: + raise ValueError(f"Wind turbine configuration '{turbine}' not found.") + + speed = self.estimate( + years=years, months=months, xs=xs, ys=ys, height=turbineconf["hub_height"] + ) + + interp_fn = interp1d( + turbineconf["V"], + turbineconf["POW"], + bounds_error=False, + fill_value="extrapolate", + ) + + # Calculate the power output + power = xr.apply_ufunc( + interp_fn, + speed, + vectorize=True, + dask="parallelized", + output_dtypes=[float], + ) + + return xr.Dataset({"cf": power / turbineconf["P"]}) diff --git a/src/geodata/model/wind/extrapolate.py b/src/geodata/model/wind/extrapolate.py index 37a4cb03..17ac008e 100644 --- a/src/geodata/model/wind/extrapolate.py +++ b/src/geodata/model/wind/extrapolate.py @@ -93,7 +93,7 @@ class WindExtrapolationModel(WindBaseModel): SUPPORTED_WEATHER_DATA_CONFIGS = {"slv_flux_hourly"} - def _prepare_fn( + def _prepare_dataset( self, ds: xr.Dataset, compute_lml: bool = True, @@ -191,41 +191,3 @@ def _estimate_dataset( result = alpha * np.log((height - ds["disph"]) / np.exp(-beta / alpha)) return result.drop_vars("coeff") # remove unnecessary coordinate - - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.DataArray: - assert height > 0, "Height must be greater than 0." - - if months is None: - months = slice(1, 12) - - start_time = pd.Timestamp(year=years.start, month=months.start, day=1) - end_time = pd.Timestamp( - year=years.stop, month=months.stop, day=31, hour=23, minute=59, second=59 - ) - - ds = xr.open_mfdataset(self.files) - - if xs is None: - xs = ds.coords["x"] - if ys is None: - ys = ds.coords["y"] - - ds = ds.sel(x=xs, y=ys, time=slice(start_time, end_time)) - - if height in HEIGHTS.values() and use_real_data: - logger.info("Using real data for estimation at height %d", height) - return (ds[f"u{height}m"] ** 2 + ds[f"v{height}m"] ** 2) ** 0.5 - - alpha = ds["coeffs"][..., 0] - beta = ds["coeffs"][..., 1] - - result = alpha * np.log((height - ds["disph"]) / np.exp(-beta / alpha)) - return result.drop_vars("coeff") # remove unnecessary coordinate diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index 2e63e490..eba93d19 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -1,4 +1,4 @@ -# Copyright 2024 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,7 +14,7 @@ # along with this program. If not, see . import logging -from typing import Hashable, Optional +from typing import Hashable import numpy as np import scipy.interpolate as sinterp @@ -22,7 +22,7 @@ from xarray.namedarray.pycompat import array_type from ...logging import logger -from ...utils import get_daterange +from ...utils import rechunk_dataset from ._base import WindBaseModel # See https://confluence.ecmwf.int/display/UDOC/L137+model+level+definitions @@ -84,7 +84,7 @@ def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: from dask.array import map_blocks from dask.diagnostics import ProgressBar - logger.info("Computing interpolation coefficients using Dask.") + logger.debug("Computing interpolation coefficients using Dask.") if len(a.data.chunks[0]) > 1: a = a.chunk({dim: -1}) @@ -122,6 +122,24 @@ def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: ) +def _splev_ker(c: np.ndarray, t: np.ndarray, k: int, height: np.ndarray) -> np.ndarray: + return np.atleast_1d(sinterp.splev(height, (t, c, k))) + + +def _splev(da: xr.DataArray, height: float) -> xr.DataArray: + height = np.atleast_1d(height) + return xr.apply_ufunc( + _splev_ker, + da["c"], + input_core_dims=[["height"]], + output_core_dims=[[]], + vectorize=True, + dask="parallelized", + output_dtypes=[da["c"].dtype], + kwargs={"t": da.attrs["t"], "k": da.attrs["k"], "height": height}, + ) + + class WindInterpolationModel(WindBaseModel): """Wind speed estimation based on a spline interpolation of the wind speed at different heights. @@ -140,7 +158,7 @@ class WindInterpolationModel(WindBaseModel): SUPPORTED_WEATHER_DATA_CONFIGS = {"wind_3d_hourly"} - def _prepare_fn( + def _prepare_dataset( self, ds: xr.Dataset, half_precision: bool = True, @@ -171,87 +189,13 @@ def _prepare_fn( logger.debug("Shape of heights: %s", ds["height"].shape) speeds = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + params = _splrep(speeds, "height") if half_precision: - speeds = speeds.astype(np.float32) - - return _splrep(speeds, "height") - - def _estimate_dataset( - self, - height: int, - years: Optional[slice] = None, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.Dataset: - params = xr.open_mfdataset(self.files).transpose("height", ...) - - if not (xs is None or ys is None): - params = params.sel(latitude=ys, longitude=xs) - - if not (years is None and months is None): - if months is None: - months = slice(1, 13) - params = params.sel( - valid_time=get_daterange(years, months), - ) - - # If the height is in the list of known heights, we can directly return - # the wind speed to save computation time. - if float(height) in LEVEL_TO_HEIGHT.values(): - params = params.sel(height=height) - return ( - ((params["u"] ** 2 + params["v"] ** 2) ** 0.5) - .drop("height") - .drop("model_level") - ) - - params = params[["c"]] - spline_params = sinterp.BSpline( - params.attrs.get("t"), params.get("c").values, k=3 - ) - - params = params.drop_dims("height") - return xr.DataArray( - spline_params(height), dims=params.dims, coords=params.coords - ) + params = params.astype(np.float32) - def _estimate_cutout( - self, - height: int, - years: slice, - months: Optional[slice] = None, - xs: Optional[slice] = None, - ys: Optional[slice] = None, - use_real_data: Optional[bool] = False, - ) -> xr.Dataset: - params = xr.open_mfdataset(self.files).transpose("height", ...) - - if not (xs is None or ys is None): - params = params.sel(latitude=ys, longitude=xs) - - if not (years is None and months is None): - if months is None: - months = slice(1, 13) - params = params.sel( - valid_time=get_daterange(years, months), - ) - - if float(height) in LEVEL_TO_HEIGHT.values() and use_real_data: - params = params.sel(height=height) - return ( - ((params["u"] ** 2 + params["v"] ** 2) ** 0.5) - .drop("height") - .drop("model_level") - ) - - params = params[["c"]] - spline_params = sinterp.BSpline( - params.attrs.get("t"), params.get("c").values, k=3 - ) + return params - params = params.drop_dims("height") - return xr.DataArray( - spline_params(height), dims=params.dims, coords=params.coords - ) + def _estimate_dataset(self, params: xr.Dataset, height: float) -> xr.Dataset: + params = params.transpose("height", ...) + params = rechunk_dataset(params, force_full_chunk_dims=["height"]) + return _splev(params, height) diff --git a/src/geodata/plot.py b/src/geodata/plot.py index 11e16f04..f6e47872 100644 --- a/src/geodata/plot.py +++ b/src/geodata/plot.py @@ -479,3 +479,6 @@ def save_animation(file_name: str): """ ) return javascript + + +__all__ = ["time_series", "heatmap", "heatmap_animation", "save_animation"] diff --git a/src/geodata/preparation.py b/src/geodata/preparation.py index 3a902be0..8370fb06 100644 --- a/src/geodata/preparation.py +++ b/src/geodata/preparation.py @@ -1,4 +1,5 @@ # Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,7 +15,7 @@ # along with this program. If not, see . -"""Various Preparation Functions for Geodata's modules""" +"""Various Preparation Functions for Geodata's modules.""" import calendar import logging @@ -22,27 +23,34 @@ import shutil import subprocess import tempfile -from glob import glob from multiprocessing import Pool +from typing import TYPE_CHECKING import dask import numpy as np import pandas as pd import xarray as xr +if TYPE_CHECKING: + from geodata.cutout import Cutout +else: + Cutout = object + logger = logging.getLogger(__name__) -def cutout_do_task(task, write_to_file=True): +def cutout_do_task(task: dict, write_to_file: bool = True): task = task.copy() prepare_func = task.pop("prepare_func") + if write_to_file: - datasetfns = task.pop("datasetfns") + datasetfns = task.pop("filenames") # Force dask to use just one thread (to save memory) with dask.config.set(scheduler="single-threaded"): try: data = prepare_func(**task) + if data is None: data = [] @@ -50,8 +58,10 @@ def cutout_do_task(task, write_to_file=True): for yearmonth, ds in data: if ds is None: continue - fn = datasetfns[yearmonth] # type: ignore + + fn = datasetfns[yearmonth] logger.debug("Writing to %s", os.path.basename(fn)) + ds.to_netcdf(fn) logger.debug( "Write variable(s) %s to %s generated by %s", @@ -59,8 +69,10 @@ def cutout_do_task(task, write_to_file=True): os.path.basename(fn), prepare_func.__name__, ) - else: - return data + return + + return data + except Exception as e: logger.exception( "Exception occured in the task with prepare_func `%s`: %s", @@ -70,19 +82,32 @@ def cutout_do_task(task, write_to_file=True): raise e -def cutout_prepare(cutout, overwrite=False, nprocesses=None, gebco_height=False): - """ - Main preparation function +def cutout_prepare( + cutout: Cutout, + overwrite: bool = False, + nprocesses: int | None = None, + gebco_height: bool = False, +): + """Main preparation function for a cutout. + + Args: + cutout: The cutout object to prepare. + overwrite: If True, the cutout will be prepared even if it is already prepared. + nprocesses: Number of processes to use. If None, all processors will be used. + gebco_height: If True, the function will interpolate the GEBCO bathymetry to the dataset grid. """ + if cutout.prepared and not overwrite: logger.info( - "The cutout is already prepared. If you want to recalculate it, supply an `overwrite=True` argument." + "The cutout is already prepared. If you want to recalculate it" + ", supply an `overwrite=True` argument." ) return True - if cutout.empty is True: + if cutout.empty: logger.warning( - "Cutout dimensions are empty. One or more of xs, ys, or time are not available in Dataset." + "Cutout dimensions are empty. One or more of xs, ys, or time are not " + "available in Dataset." ) return False @@ -93,109 +118,68 @@ def cutout_prepare(cutout, overwrite=False, nprocesses=None, gebco_height=False) xs = cutout.meta.indexes["x"] ys = cutout.meta.indexes["y"] - # 1. (here) uncomment and change yearmonths to only new ones. delete yearmonths line above - # 2. (elsewhere) uncomment meta_append / append lines - - # if cutout.meta_append == 1: - # # appended meta. prepare only new year-months - # yearmonths = cutout.coords['year-month'].to_index() - # else: - # yearmonths = cutout.coords['year-month'].to_index() - if gebco_height: logger.info("Interpolating gebco to the dataset grid") cutout.meta["height"] = _prepare_gebco_height(xs, ys) # Check if cutout_dir exists - if os.path.isdir(cutout_dir): + if cutout_dir.is_dir(): # Delete all files except meta.nc - logger.debug("Deleting cutout files in '%s'", cutout_dir) - for delete_file in glob(os.path.join(cutout_dir, "*.*")): - if not delete_file.endswith("meta.nc"): - logger.debug(delete_file) - os.remove(delete_file) - # shutil.rmtree(cutout_dir) - else: - os.mkdir(cutout_dir) - - # # Moved to cutout __init__: Write meta file - # (cutout.meta_clean - # .unstack('year-month') - # .to_netcdf(cutout.datasetfn())) + for f in cutout_dir.rglob("*.nc"): + if not f.name.endswith("meta.nc"): + logger.debug("Deleting %s", f.name) + f.unlink() + cutout_dir.mkdir(parents=True, exist_ok=True) # Compute data and fill files - tasks = [] - - # for series in itervalues(cutout.weather_data_config[cutout.config]): - # dict of tasks w/structure (tasks_func, prepare_func) - # .. could be one task and prepare (eg prepare_month_era5) - series = cutout.weather_data_config[cutout.config] - series["meta_attrs"] = cutout.meta.attrs - tasks_func = series["tasks_func"] + tasks = cutout.dataset_cls.tasks_func(xs=xs, ys=ys, yearmonths=yearmonths) - # form call to task_func (eg tasks_monthly_merra2) - # .. **series contains prepare_func - # returns: dict(prepare_func=prepare_func, xs=xs, ys=ys, year=year, month=month) - # .. or: dict(prepare_func=prepare_func, xs=xs, ys=ys, fn=next(glob...), engine=engine, yearmonth=ym) - tasks += tasks_func(xs=xs, ys=ys, yearmonths=yearmonths, **series) for i, t in enumerate(tasks): def datasetfn_with_id(ym): # returns a filename with incrementing id at end eg `201101-01.nc` - base, ext = os.path.splitext(cutout.datasetfn(ym)) - return f"{base}-{i}{ext}" - - t["datasetfns"] = {ym: datasetfn_with_id(ym) for ym in yearmonths.tolist()} - - # TODO: Using multiple processes will cause issues with geodata currently - # because geodata write the metadata dataframe to the cutout directory - # in every instantiation of a new Cutout object in an unprepared state. - # By running multiple processes, the metadata file will be overwritten - # and there are no locking mechanism currently in place. - # As the result, we will run the tasks in a single process for now. - # In the future, we can consider doing it in a multiprocessing scheme as long as - # we can ensure that the metadata file is being processed in places other than - # the constructor. - - # logger.info( - # "%d tasks have been collected. Starting running them on %s.", - # len(tasks), - # f"{nprocesses} processes" if nprocesses is not None else "all processors", - # ) - - pool = Pool(processes=1) - try: - pool.map(cutout_do_task, tasks) - except Exception as e: - pool.terminate() - logger.info( - "Preparation of cutout '%s' has been interrupted by an exception. " - "Purging the incomplete cutout_dir.", - cutout.name, - ) - shutil.rmtree(cutout_dir) - raise e - pool.close() + filename = cutout._get_filename(ym) + return filename.with_stem(f"{filename.stem}-{i:02d}") + + t["filenames"] = {ym: datasetfn_with_id(ym) for ym in yearmonths.tolist()} + + logger.info( + "%d tasks have been collected. Starting running them on %s.", + len(tasks), + f"{nprocesses} processes" if nprocesses is not None else "all processors", + ) + + with Pool(processes=nprocesses) as pool: + try: + pool.map(cutout_do_task, tasks) + except Exception as e: + pool.terminate() + logger.warning( + "Preparation of cutout '%s' has been interrupted by an exception. " + "Purging the incomplete cutout_dir.", + cutout.name, + ) + shutil.rmtree(cutout_dir) + raise e logger.info("Merging variables into monthly compound files") - for fn in map(cutout.datasetfn, yearmonths.tolist()): + for fn in [cutout._get_filename(ym) for ym in yearmonths.tolist()]: # Find all files with yearmonth prefix eg `201101-XX.nc` - base, ext = os.path.splitext(fn) - fns = glob(base + "-*" + ext) + fns = list(cutout_dir.rglob(f"{fn.stem}-*{fn.suffix}")) + if len(fns) == 1 and not gebco_height: # Just a single file. Simply rename - os.rename(fns[0], fn) + fns[0].rename(fn) else: - # Multiple files for yearmonth - # open_mfdataset: auto-magically determines appropriate concat and merge of datasets - with xr.open_mfdataset(fns, combine="by_coords") as ds: + with xr.open_mfdataset(fns, combine="by_coords", chunks="auto") as ds: if gebco_height: ds["height"] = cutout.meta["height"] ds.to_netcdf(fn) - for tfn in fns: - os.unlink(tfn) + for f in fns: + f.unlink() + logger.debug("Completed files %s", os.path.basename(fn)) logger.info("Cutout '%s' has been successfully prepared", cutout.name) @@ -216,102 +200,26 @@ def cutout_produce_specific_dataseries(cutout, yearmonth, series_name): return data[0][1] # type: ignore -def cutout_get_meta(cutout, xs, ys, years, months=None, **dataset_params): - # called in cutout.py as `get_meta()` - # Loads various metadata (coordinates, dims...) from dataset via dataset_module.prepare_func (eg prepare_meta_merra2) - - if months is None: - months = slice(1, 12) - - ys = _prepare_lat_direction(cutout.dataset_module.lat_direction, ys) - - meta_kwds = cutout.meta_data_config.copy() - meta_kwds.update(dataset_params) - - # Assign task function here? - tasks_func = meta_kwds["tasks_func"] # noqa: F841 - # test before removing - - # Get metadata - prepare_func = meta_kwds.pop("prepare_func") - ds = prepare_func(xs=xs, ys=ys, year=years.stop, month=months.stop, **meta_kwds) - ds.attrs.update(dataset_params) - - # Check if cutout dimenions are empty - dims = [len(ds.indexes[i]) for i in ds.indexes] - if np.prod(dims) == 0: - logger.warning( - "Cutout dimensions are empty. One or more of xs, ys, or time are not available in Dataset." - ) - cutout.empty = True - - # with metadata, load various parameters - meta_file_granularity = meta_kwds["file_granularity"] - month_start = pd.Timestamp(f"{years.stop}-{months.stop}") - ds.coords["year"] = range(years.start, years.stop + 1) - ds.coords["month"] = range(months.start, months.stop + 1) - - if meta_file_granularity == "daily": - start, second, end = map(pd.Timestamp, ds.coords["time"].values[[0, 1, -1]]) - offset_start = start - month_start - offset_end = end - (month_start + pd.offsets.MonthBegin()) - step = (second - start).components.hours - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}") + offset_start, - end=(month_start + pd.offsets.MonthBegin() + offset_end), - freq="h" if step == 1 else f"{step}h", - ) - elif meta_file_granularity == "dailymeans": - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}-1"), - end=pd.Timestamp( - f"{years.stop}-{months.stop}-{calendar.monthrange(years.stop, months.stop)[1]}" - ), - freq="d", - ) - elif meta_file_granularity == "monthly": - ds.coords["time"] = pd.date_range( - start=pd.Timestamp(f"{years.start}-{months.start}"), - end=pd.Timestamp(f"{years.stop}-{months.stop}"), - freq="MS", - ) - - ds = ds.stack(**{"year-month": ("year", "month")}) - - # if cutout.meta_append == 1: - # # Append to existing meta - # (ds - # .unstack('year-month') - # .to_netcdf(cutout.datasetfn('meta','2')) ) - # - # with xr.open_mfdataset([cutout.datasetfn(),cutout.datasetfn('meta','2')], combine='by_coords') as ds_comb: - # ds = ds_comb - # ds = ds.stack(**{'year-month': ('year', 'month')}) - - return ds - - def cutout_get_meta_view( - cutout, xs=None, ys=None, years=slice(None), months=slice(None), **dataset_params + cutout: Cutout, + xs: slice | None = None, + ys: slice | None = None, + years=slice(None), + months=slice(None), + **dataset_params, ): - # called in cutout as `get_meta_view()` - # Create subset of metadata based on xs, ys, years, months - # Returns None if any of the dimensions of the subset are empty - meta = cutout.meta meta.attrs["view"] = {} if xs is not None: - meta.attrs.setdefault("view", {})["x"] = xs + meta.attrs["x"] = xs if ys is not None: - meta.attrs.setdefault("view", {})["y"] = _prepare_lat_direction( - cutout.dataset_module.lat_direction, ys - ) + meta.attrs["y"] = _prepare_lat_direction(cutout.dataset_cls.lat_direction, ys) meta = ( meta.unstack("year-month") .sel(year=years, month=months, **meta.attrs.get("view", {})) - .stack(**{"year-month": ("year", "month")}) + .stack(dim={"year-month": ("year", "month")}) ) meta = meta.sel( @@ -328,9 +236,73 @@ def cutout_get_meta_view( dim_len = [len(meta.indexes[i]) for i in meta.indexes] if all(d > 0 for d in dim_len): return meta - else: - logger.info(dim_len) - return None + + logger.warning( + "Certain dimensions are empty with the meta file of the cutout %s", dim_len + ) + + +def cutout_get_meta( + cutout: Cutout, + xs: slice, + ys: slice, + years: slice, + months: slice = slice(1, 12), + **dataset_params, +) -> xr.Dataset: + ys = _prepare_lat_direction(cutout.dataset_cls.lat_direction, ys) + + meta_kwds = cutout.meta_data_config.copy() + meta_kwds.update(dataset_params) + + # Get metadata + prepare_func: callable = meta_kwds.pop("prepare_func") + ds: xr.Dataset = prepare_func( + xs=xs, ys=ys, year=years.stop, month=months.stop, **meta_kwds + ) + ds.attrs.update(dataset_params) + + # Check if cutout dimenions are empty + if not np.prod([len(ds.indexes[i]) for i in ds.indexes]): + logger.warning( + "Cutout dimensions are empty. " + "One or more of xs, ys, or time are not available in Dataset." + ) + cutout.empty = True + + # with metadata, load various parameters + meta_file_granularity = meta_kwds["file_granularity"] + month_start = pd.Timestamp(f"{years.stop}-{months.stop}") + ds.coords["year"] = range(years.start, years.stop + 1) + ds.coords["month"] = range(months.start, months.stop + 1) + + match meta_file_granularity: + case "daily": + start, second, end = map(pd.Timestamp, ds.coords["time"].values[[0, 1, -1]]) + offset_start = start - month_start + offset_end = end - (month_start + pd.offsets.MonthBegin()) + step = (second - start).components.hours + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}") + offset_start, + end=(month_start + pd.offsets.MonthBegin() + offset_end), + freq="h" if step == 1 else f"{step}h", + ) + case "dailymeans": + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}-1"), + end=pd.Timestamp( + f"{years.stop}-{months.stop}-{calendar.monthrange(years.stop, months.stop)[1]}" + ), + freq="d", + ) + case "monthly": + ds.coords["time"] = pd.date_range( + start=pd.Timestamp(f"{years.start}-{months.start}"), + end=pd.Timestamp(f"{years.stop}-{months.stop}"), + freq="MS", + ) + + return ds.stack(**{"year-month": ("year", "month")}) def _prepare_gebco_height(xs, ys, gebco_fn=None): @@ -387,11 +359,12 @@ def _prepare_gebco_height(xs, ys, gebco_fn=None): return height -def _prepare_lat_direction(lat_direction, ys): +def _prepare_lat_direction(lat_direction: bool, ys: slice): # Check direction of latitudes encoded in dataset, flip if necessary if not lat_direction and ys.stop > ys.start: ys = slice(ys.stop, ys.start, -ys.step if ys.step is not None else None) if lat_direction and ys.stop < ys.start: ys = slice(ys.stop, ys.start, ys.step if ys.step is not None else None) + return ys diff --git a/src/geodata/resource.py b/src/geodata/resource.py index 7bbcd962..86a12ad3 100644 --- a/src/geodata/resource.py +++ b/src/geodata/resource.py @@ -23,6 +23,7 @@ import logging from operator import itemgetter + import numpy as np import yaml from scipy.signal import fftconvolve @@ -32,14 +33,34 @@ logger = logging.getLogger(name=__name__) -def get_windturbineconfig(turbine): - """Load the 'turbine'.yaml file from local disk and provide a turbine dict.""" +def get_windturbineconfig(turbine: str): + """Load the 'turbine'.yaml file from local disk and provide a turbine dict. + + Args: + turbine (str): Name of the wind turbine configuration file (without .yaml). + + Returns: + dict: Dictionary containing the wind turbine configuration. + - V: Wind speed array + - POW: Power output array + - hub_height: Hub height + - P: Rated power output + """ res_name = "resources/windturbine/" + turbine + ".yaml" with open(SRC_ROOT / res_name, "r") as resource_file: turbineconf = yaml.safe_load(resource_file) V, POW, hub_height = itemgetter("V", "POW", "HUB_HEIGHT")(turbineconf) - return dict(V=np.array(V), POW=np.array(POW), hub_height=hub_height, P=np.max(POW)) + + # Let's make sure that the wind speed and power output arrays are sorted + V = np.asarray(V) + POW = np.asarray(POW) + + sorted_indices = np.argsort(V) + V = V[sorted_indices] + POW = POW[sorted_indices] + + return {"V": V, "POW": POW, "hub_height": hub_height, "P": POW.max()} def get_solarpanelconfig(panel): @@ -140,3 +161,15 @@ def smooth(velocities, power): ) return turbine + + +def get_available_windturbines(): + """Get a list of available wind turbine configurations.""" + res_path = SRC_ROOT / "resources/windturbine" + return [p.stem for p in res_path.glob("*.yaml") if p.is_file()] + + +def get_available_solarpanels(): + """Get a list of available solar panel configurations.""" + res_path = SRC_ROOT / "resources/solarpanel" + return [p.stem for p in res_path.glob("*.yaml") if p.is_file()] diff --git a/src/geodata/types.py b/src/geodata/types.py new file mode 100644 index 00000000..9bf5715f --- /dev/null +++ b/src/geodata/types.py @@ -0,0 +1,28 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""This module defines some of the common types used in the geodata package.""" + +from pathlib import Path + +# Geodata Specific Types +DateRange = slice | tuple[int, int] | list[int] +CoordRange = slice | tuple[float, float] | list[float] +BoundRange = tuple[float, float, float, float] | list[float] + +# Filesystem +PathLike = str | Path + +__all__ = ["DateRange", "CoordRange", "BoundRange"] diff --git a/src/geodata/utils.py b/src/geodata/utils.py index 21b362e4..b8aae82f 100644 --- a/src/geodata/utils.py +++ b/src/geodata/utils.py @@ -1,4 +1,4 @@ -# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023, 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,35 +14,14 @@ # along with this program. If not, see . +import hashlib import json +from pathlib import Path +import dask.array as da import numpy as np import pandas as pd -import progressbar as pgb - - -def make_optional_progressbar(show, prefix, max_value): - if show: - widgets = [ - pgb.widgets.Percentage(), - " ", - pgb.widgets.SimpleProgress(), - " ", - pgb.widgets.Bar(), - " ", - pgb.widgets.Timer(), - " ", - pgb.widgets.ETA(), - ] - if not prefix.endswith(": "): - prefix = prefix.strip() + ": " - maybe_progressbar = pgb.ProgressBar( - prefix=prefix, widgets=widgets, max_value=max_value - ) - else: - maybe_progressbar = lambda x: x # noqa: E731 - - return maybe_progressbar +import xarray as xr def dummy_njit(f=None, *args, **kwargs): @@ -69,7 +48,7 @@ def get_daterange(years: slice, months: slice): months (slice): The months range. Returns: - pd. + pd.DatetimeIndex: The date range. """ assert years.start <= years.stop, "Start year must be less than stop year." @@ -95,3 +74,109 @@ def default(self, obj): if isinstance(obj, np.ndarray): return obj.tolist() return super(NpEncoder, self).default(obj) + + +def ensure_slice(obj: slice | list): + """Ensure that the input is a slice object. If the input is a list, convert it to a slice object. + + Args: + obj (slice | list): The input object. + + Returns: + slice: The converted slice object. + """ + if isinstance(obj, list) and (len(obj) == 2 or len(obj) == 3): + return slice(*obj) + elif isinstance(obj, slice): + return obj + else: + raise TypeError("Input must be a slice or a list.") + + +def check_hash(file: Path, saved_hash: str | None = None) -> tuple[bool, str]: + """Check if the hash of a file matches the given hash. + + Args: + file (Path): The path to the file. + saved_hash (str | None): The hash to compare against. If None, the function will compute the hash of the file. + If provided, the function will compare the computed hash with this value. + If the hashes match, the function will return True and the computed hash. + If the hashes do not match, the function will return False and the computed hash. + + Returns: + tuple[bool, str]: A tuple containing a boolean indicating if the hash matches and the computed hash. + """ + + if not file.exists(): + return False, "" + + with open(file, "rb") as f: + computed_hash = hashlib.sha256(f.read()).hexdigest() + + if saved_hash is None: + return True, computed_hash + else: + return computed_hash == saved_hash, computed_hash + + +def rechunk_dataset( + data: xr.DataArray | xr.Dataset, + target_chunk_bytes: int = 20 * 1024**2, + force_full_chunk_dims: list[str] | None = None, +): + """ + Rechunk xarray DataArray or Dataset to maximize chunk size + under a memory limit, while forcing certain dimensions to be unchunked + (i.e., use only one chunk across that dimension). + + Parameters: + data: xr.DataArray or xr.Dataset + target_chunk_bytes: maximum memory per chunk (in bytes) + force_full_chunk_dims: list of dimension names to not chunk (single chunk along that dim) + """ + if force_full_chunk_dims is None: + force_full_chunk_dims = [] + + if isinstance(data, xr.Dataset): + vars_to_chunk = { + name: rechunk_dataset(var, target_chunk_bytes, force_full_chunk_dims) + for name, var in data.data_vars.items() + } + return data.assign(vars_to_chunk) + + if not isinstance(data.data, da.Array): + raise ValueError("Data must be a Dask-backed xarray object") + + shape = data.shape + dims = data.dims + itemsize = data.dtype.itemsize + + # Start with full dims + chunk_shape = list(shape) + dim_to_index = {dim: i for i, dim in enumerate(dims)} + + # Force full chunks on specified dims + for dim in force_full_chunk_dims: + if dim in dim_to_index: + chunk_shape[dim_to_index[dim]] = shape[dim_to_index[dim]] + + # Reduce non-fixed dims to fit memory budget + while True: + est_bytes = np.prod(chunk_shape) * itemsize + if est_bytes <= target_chunk_bytes: + break + + # Pick largest non-fixed dimension to halve + candidates = [ + (i, size) + for i, size in enumerate(chunk_shape) + if dims[i] not in force_full_chunk_dims and size > 1 + ] + if not candidates: + break # Can't reduce further + + i, _ = max(candidates, key=lambda x: x[1]) + chunk_shape[i] = max(1, chunk_shape[i] // 2) + + chunk_dict = dict(zip(dims, chunk_shape)) + return data.chunk(chunk_dict) diff --git a/tests/pr/test_era5.py b/tests/pr/test_era5.py deleted file mode 100644 index 7c4ad894..00000000 --- a/tests/pr/test_era5.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2022 Xiqiang Liu - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -from itertools import product - -import numpy as np -import xarray as xr - -import geodata - -logging.basicConfig(level=logging.INFO) - - -def get_data_configs() -> list[str]: - return ["wind_solar_hourly"] - - -def get_bounds() -> list[list[int]]: - return [[50, 0, 48, 3]] - - -def get_years() -> list[slice]: - return [slice(2005, 2005)] - - -def get_months() -> list[slice]: - return [slice(1, 1)] - - -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - -def get_turbine() -> str: - return "Suzlon_S82_1.5_MW" - - -def get_smooth() -> bool: - return True - - -def get_panel() -> str: - return "KANEKA" - - -def get_orientation() -> str: - return "latitude_optimal" - - -def get_era5(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="era5", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, - ) - if not dataset.prepared: - dataset.get_data(testing=True) - return dataset - - -def create_cutout(data_config: str, x: slice, y: slice, year: slice, month: slice): - cutout = geodata.Cutout( - name=f"era-{data_config}-{year.start}-{month.start}", - module="era5", - weather_data_config=data_config, - xs=x, - ys=y, - years=year, - months=month, - ) - cutout.prepare() - return cutout - - -def create_wind_output(cutout: geodata.Cutout, turbine, smooth): - ds_wind = cutout.wind(turbine, smooth) - df_wind = ds_wind.to_dataframe(name="wind") - return df_wind - - -def create_windspd_output(cutout: geodata.Cutout, turbine): - ds_windspd = cutout.wind(turbine) - df_windspd = ds_windspd.to_dataframe(name="windspd") - return df_windspd - - -def create_pv_output(cutout: geodata.Cutout, panel, orientation): - ds_pv = cutout.pv(panel, orientation) - df_pv = ds_pv.to_dataframe(name="pv") - return df_pv - - -def test_download(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in product(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] - - -def test_cutout(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - assert cutout.prepared - - -def test_wind_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "wind": np.floating, - } - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - smooth = get_smooth() - df_wind = create_wind_output(cutout, turbine, smooth) - - for col in supposed_dtypes: - assert np.issubdtype(df_wind[col].dtype, supposed_dtypes[col]) - - -def test_windspd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "windspd": np.floating, - } - - for config, year, month, x, y in product(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - df_windspd = create_windspd_output(cutout, turbine) - - for col in supposed_dtypes: - assert np.issubdtype(df_windspd[col].dtype, supposed_dtypes[col]) - - -def test_pv_output(): - configs = [config for config in get_data_configs() if "hourly" in config] - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - supposed_dtypes = { - "lat": np.floating, - "lon": np.floating, - "pv": np.floating, - } - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - panel = get_panel() - orientation = get_orientation() - df_pv = create_pv_output(cutout, panel, orientation) - - for col in supposed_dtypes: - assert np.issubdtype(df_pv[col].dtype, supposed_dtypes[col]) diff --git a/tests/pr/test_era5_lengthy.py b/tests/pr/test_era5_lengthy.py index 8fdcd37a..9979ace3 100644 --- a/tests/pr/test_era5_lengthy.py +++ b/tests/pr/test_era5_lengthy.py @@ -17,9 +17,7 @@ import logging -import geodata - -import xarray as xr +from geodata.datasets import DatasetType, load_dataset logging.basicConfig(level=logging.INFO) @@ -41,24 +39,13 @@ def get_months() -> list[slice]: return [slice(1, 2)] -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - def get_era5(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="era5", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, + dataset_cls = load_dataset(data_config) + dataset: DatasetType = dataset_cls( + years=year, months=month, bounds=bound, testing=True ) - if not dataset.prepared: - dataset.get_data() + if not dataset.downloaded: + dataset.download() return dataset @@ -70,19 +57,4 @@ def test_download(): for config, year, month, bound in zip(configs, years, months, bounds): dataset = get_era5(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_era5(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] + assert dataset.downloaded diff --git a/tests/pr/test_mask.py b/tests/pr/test_mask.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/pr/test_merra2.py b/tests/pr/test_merra2.py index 27f8569a..ffff5169 100644 --- a/tests/pr/test_merra2.py +++ b/tests/pr/test_merra2.py @@ -15,10 +15,7 @@ import logging -import geodata - -import xarray as xr -import numpy as np +from geodata.datasets import DatasetType, load_dataset logging.basicConfig(level=logging.INFO) @@ -44,71 +41,16 @@ def get_months() -> list[slice]: return [slice(1, 1)] -def get_xs() -> list[slice]: - return [slice(48.5, 49.5)] - - -def get_ys() -> list[slice]: - return [slice(1, 2.5)] - - -def get_turbine() -> str: - return "Suzlon_S82_1.5_MW" - - -def get_smooth() -> bool: - return True - - -def get_var_height() -> str: - return "lml" - - def get_merra2(data_config: str, bound: list[int], year: slice, month: slice): - dataset = geodata.Dataset( - module="merra2", - weather_data_config=data_config, - years=year, - months=month, - bounds=bound, + dataset_cls = load_dataset(data_config) + dataset: DatasetType = dataset_cls( + years=year, months=month, bounds=bound, testing=True ) - if not dataset.prepared: - dataset.get_data(testing=True) + if not dataset.downloaded: + dataset.download() return dataset -def create_cutout(data_config: str, x: slice, y: slice, year: slice, month: slice): - cutout = geodata.Cutout( - name=f"merra2-{data_config}-{year.start}-{month.start}", - module="merra2", - weather_data_config=data_config, - xs=x, - ys=y, - years=year, - months=month, - ) - cutout.prepare() - return cutout - - -def create_wind_output(cutout: geodata.Cutout, turbine, smooth, var_height): - ds_wind = cutout.wind(turbine, smooth, var_height=var_height) - df_wind = ds_wind.to_dataframe(name="wind") - return df_wind - - -def create_windspd_output(cutout: geodata.Cutout, turbine, var_height): - ds_windspd = cutout.windspd(turbine=turbine, var_height=var_height) - df_windspd = ds_windspd.to_dataframe(name="windspd") - return df_windspd - - -def create_windwpd_output(cutout: geodata.Cutout, turbine, var_height): - ds_windwpd = cutout.windwpd(turbine=turbine, var_height=var_height) - df_windwpd = ds_windwpd.to_dataframe(name="windwpd") - return df_windwpd - - def test_download(): configs = get_data_configs() years = get_years() @@ -117,89 +59,4 @@ def test_download(): for config, year, month, bound in zip(configs, years, months, bounds): dataset = get_merra2(config, bound, year, month) - assert dataset.prepared - - -def test_trim(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() - - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_merra2(config, bound, year, month) - dataset.trim_variables() - for f in dataset.downloadedFiles: - file_path = f[1] - with xr.open_dataset(file_path) as ds: - assert list(ds.data_vars) == dataset.weatherconfig["variables"] - - -def test_cutout(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - assert cutout.prepared - - -def test_wind_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - smooth = get_smooth() - var_height = get_var_height() - df_wind = create_wind_output(cutout, turbine, smooth, var_height) - assert df_wind.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "wind": np.dtype("float64"), - } - - -def test_windspd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - var_height = get_var_height() - df_windspd = create_windspd_output(cutout, turbine, var_height) - assert df_windspd.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "windspd": np.dtype("float32"), - } - - -def test_windwpd_output(): - configs = get_data_configs() - years = get_years() - months = get_months() - xs = get_xs() - ys = get_ys() - - for config, year, month, x, y in zip(configs, years, months, xs, ys): - cutout = create_cutout(config, x, y, year, month) - turbine = get_turbine() - var_height = get_var_height() - df_windwpd = create_windwpd_output(cutout, turbine, var_height) - assert df_windwpd.dtypes.to_dict() == { - "lon": np.dtype("float64"), - "lat": np.dtype("float64"), - "windwpd": np.dtype("float32"), - } + assert dataset.downloaded diff --git a/uv.lock b/uv.lock index 7e5888c6..64bcef3f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,4 +1,5 @@ version = 1 +revision = 1 requires-python = ">=3.10" resolution-markers = [ "python_full_version < '3.11'", @@ -275,33 +276,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/96/b100d19b55acc2b3cf41c52f31804533534de6e9da445ba11d83fa262d5d/Bottleneck-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1339b9ad3ee217253f246cde5c3789eb527cf9dd31ff0a1f5a8bf7fc89eadad", size = 111760 }, ] -[[package]] -name = "cads-api-client" -version = "1.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "multiurl" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/d5/54e3f4153e07b053b4e3b1576fc385c33263f40ca5030d9b3ef07013d4d0/cads_api_client-1.4.3.tar.gz", hash = "sha256:52359ea743a84b597cc589730ff4b414707bce4e5e2279835528b2d1273c19b8", size = 40795 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/87/bece1e563ce5602a2cd6b92e6b870fa8ad45a9c21866e31326385c766efd/cads_api_client-1.4.3-py3-none-any.whl", hash = "sha256:8f2db934da8f2c63902bf808f33eb2e76734feed284783acd2086a72dd4d6833", size = 24332 }, -] - [[package]] name = "cdsapi" -version = "0.7.3" +version = "0.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cads-api-client" }, + { name = "datapi" }, { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/50/b8/aaade2b628ea5784f637a0c2762565a0fb9f0b121a890aa6461e06b70659/cdsapi-0.7.3.tar.gz", hash = "sha256:883a1376ca495457eb55fd548dbbb6f5b64f2e4c880b3586dd37ba9041e51c82", size = 13121 } +sdist = { url = "https://files.pythonhosted.org/packages/3b/62/81b38105ef75486308179d510360c1aa251ce0b7d17fe88c0e1de05c6c94/cdsapi-0.7.5.tar.gz", hash = "sha256:55221c573b8cefe83cc0bfe01a3d31213c82bf9acce70455350dd24b8095c23a", size = 13188 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/2d/bfeabe547fc186471b49b9df384f44c181fe9f41899d2cf13f5bda195af9/cdsapi-0.7.3-py2.py3-none-any.whl", hash = "sha256:3bf432783e6ff0b47b0b33466c6e05e7ddad52fda5f05bf269596f5be30d623b", size = 12140 }, + { url = "https://files.pythonhosted.org/packages/4e/c4/49f01f1382d449581d5d3db0fa49ccc23a1b4f91d615108b631c7cff40cd/cdsapi-0.7.5-py2.py3-none-any.whl", hash = "sha256:8586b837aea89ceeae379b388fbb0ace0a19b94b221f731c65632417007f69fb", size = 12201 }, ] [[package]] @@ -370,6 +356,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, ] +[[package]] +name = "cfgrib" +version = "0.9.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "eccodes" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/3b/0ccbbc67866a4a2df570d6bf0f53d6d22220c44e1f3684455b5eae298936/cfgrib-0.9.15.0.tar.gz", hash = "sha256:d455034e19b9560a75d008ba9d09b2d4e65762adfb2e911f28b841f4b9c6b47f", size = 6511752 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/d7/96b4209c99f1fd6c19f502cebe8c91983c23331c380f3f521250f268ae8c/cfgrib-0.9.15.0-py3-none-any.whl", hash = "sha256:469cfd25dc173863795e596263b3b6b5ea1402b1715f2b7b1d4b995b40b32c18", size = 48908 }, +] + [[package]] name = "cftime" version = "1.6.4" @@ -458,7 +459,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } wheels = [ @@ -609,6 +610,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ea/3df533d551bc673f8a295450a8e28707e980fd3b55117edb1d1aa4cc374d/dask-2024.9.1-py3-none-any.whl", hash = "sha256:3757bb6c976f0436fef6bd6ad32f8983ee5ce7d8a738a1f643e208cd390ec794", size = 1257378 }, ] +[package.optional-dependencies] +distributed = [ + { name = "distributed" }, +] + +[[package]] +name = "datapi" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "multiurl" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/95/e4abd8958a15793a9d6df1d6f0bcff2ac245901d9cc44dfb14be3bbb1bf6/datapi-0.1.1.tar.gz", hash = "sha256:7526169c61d103bd8585a54778aa234ba89ef6f6093d2079db9d089ad5e448e0", size = 43438 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/a2/8f4f500bfa3b407afdd3bd5987dd0d5f0b1dcfb9cfd88a95a753bd76c71a/datapi-0.1.1-py3-none-any.whl", hash = "sha256:3c8e9c7c5a29b9b392633d9527266080868d3f411b78e05acac4cd9304599bd9", size = 26899 }, +] + [[package]] name = "debugpy" version = "1.8.6" @@ -648,6 +669,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] +[[package]] +name = "distributed" +version = "2024.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "cloudpickle" }, + { name = "dask" }, + { name = "jinja2" }, + { name = "locket" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "sortedcontainers" }, + { name = "tblib" }, + { name = "toolz" }, + { name = "tornado" }, + { name = "urllib3" }, + { name = "zict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/da/bf22bb58ddfe518a4bc2385b5ec17a0ad1b7cd6bc2fa8f4310181da802a2/distributed-2024.9.1.tar.gz", hash = "sha256:4d573d89ff4fdde0dd96ad5cfdb843ce8ecef8caf002435bc60d14414dc1e819", size = 1113189 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ff/7531b2bebc84766905d6b6c636f21e465511f330957bc1632d00c75ac152/distributed-2024.9.1-py3-none-any.whl", hash = "sha256:f7acca78afcc50eb337fd46247a33efa5a1696b937083a12556ba33f11be1c74", size = 1020546 }, +] + +[[package]] +name = "doc8" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "pygments" }, + { name = "restructuredtext-lint" }, + { name = "stevedore" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/28/b0a576233730b756ca1ebb422bc6199a761b826b86e93e5196dfa85331ea/doc8-1.1.2.tar.gz", hash = "sha256:1225f30144e1cc97e388dbaf7fe3e996d2897473a53a6dae268ddde21c354b98", size = 27030 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/f1/6ffd5d76578e98a8f21ae7216b88a7212c778f665f1a8f4f8ce6f9605da4/doc8-1.1.2-py3-none-any.whl", hash = "sha256:e787b3076b391b8b49400da5d018bacafe592dfc0a04f35a9be22d0122b82b59", size = 25794 }, +] + [[package]] name = "docutils" version = "0.21.2" @@ -657,6 +720,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] +[[package]] +name = "eccodes" +version = "2.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi" }, + { name = "findlibs" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/76/bb33d761b1fa7a642e29c5e279feabf6f124b15c6eed64d012a30bf5c1ec/eccodes-2.39.0.tar.gz", hash = "sha256:0bf32c1f32c00d6c12091344bb5917d68f94cbc76625e66b16055784dbbc5c03", size = 2266990 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/63/33acf0c01567fdde5ea5baf92f085bdfd4961af3496476a03fdc4549a610/eccodes-2.39.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:6f1d637fe07fa2ff80954c442b9f63c3441cdadc2a561772bd3c988264ead27c", size = 6486940 }, + { url = "https://files.pythonhosted.org/packages/47/e3/b1298b31926a35f06af6ea0c4ae958b7ef5d3511c242b7fa34dd05df9dda/eccodes-2.39.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:d436eb91c21bb5697956083ca2b8b25fb5847a144ccd84391c5ffea1650f8f84", size = 6580698 }, + { url = "https://files.pythonhosted.org/packages/84/c8/d6c5eaf6a4c43b08265c6a7a67d1f4bcdae6c4bffc1102f51a902e0b6f51/eccodes-2.39.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1cd13926e94226de2517002cab401823cd469aa05baf0cae6b8ef086a44ca58a", size = 7329463 }, + { url = "https://files.pythonhosted.org/packages/b0/aa/1a5c386f13ab8df9ee787a08b0e35f619ad8b4e65b1b7e9ec252ea0f7b40/eccodes-2.39.0-cp310-cp310-win_amd64.whl", hash = "sha256:3e59e1f33e6572eae6348e5b49b6de081a7a7cd1601fa0c31273f84f29d1ae9a", size = 6157549 }, + { url = "https://files.pythonhosted.org/packages/3f/f2/7355a83be71880cc8c41d6093bf7e7fe56e75255080a692a7751e78b6f40/eccodes-2.39.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:e12720dff21c952fc5706c58639e6a4f9ab04afe549d7d9ae34a5a6975c80c4a", size = 6486941 }, + { url = "https://files.pythonhosted.org/packages/0c/51/34070e1d4ecdd2a162a351f48e50750f8677a73aabf80669fcf61a2189a0/eccodes-2.39.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:9d55059ba468cb014ecbc8df4186a34ad80ad76e840859482f01e798dcb1a6a4", size = 6580697 }, + { url = "https://files.pythonhosted.org/packages/8b/cc/64bf04c8873278bd7561ac5492651a293dbda9993c9119586edbc543c7de/eccodes-2.39.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ad098a71880ee1496db95255c5f8aac246035eed508719251e460a375d759231", size = 7329478 }, + { url = "https://files.pythonhosted.org/packages/4e/4a/e7a6b62b33c7d541906ecd54252586befe4123ad3ab126c7eda95af08476/eccodes-2.39.0-cp311-cp311-win_amd64.whl", hash = "sha256:a23091b75610981ce1c677d36d3d07ab9f646f3ede25ef12030e2ba93b57211b", size = 6157548 }, + { url = "https://files.pythonhosted.org/packages/2d/42/d8e0f51dee957c66acbf32803f086cdc1d5485090e6eb52f88450ec304c6/eccodes-2.39.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:9d49b6bda883051c0dd9b23bd8fd79849b50801139ce5841ff9b8acbc11cccef", size = 6486942 }, + { url = "https://files.pythonhosted.org/packages/1b/6f/fde312acec0fcebc6253c2e1458d39f3782bd8c92a5513287ebe7013e1ec/eccodes-2.39.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:6be54b2362f103e0a4bb2c8f7a487de52d5813e9eb0de153a1d6b88e3d306329", size = 6580698 }, + { url = "https://files.pythonhosted.org/packages/71/46/1f176bbb20c46f127dc34f260b5d633c8ca4aef9835f71407f33410430cc/eccodes-2.39.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:772e0836e4a99777c71a122abd047c11c7b0598b7f574591ebe277310952623d", size = 7329664 }, + { url = "https://files.pythonhosted.org/packages/a1/59/4c2c19b9604971309b050e19836193a79405987917430409698e7a8ff55a/eccodes-2.39.0-cp312-cp312-win_amd64.whl", hash = "sha256:976a5fbfb74612cc79742aa8d54dbfa384c090d7c6574b6360f09665938141b7", size = 6157548 }, + { url = "https://files.pythonhosted.org/packages/c2/a4/fb4c7e2f7df631ff2cfdf345f6978452c00bdc24b0c9eff33081d6d533f4/eccodes-2.39.0-py3-none-any.whl", hash = "sha256:39f63f09a93b33dfdd805a41b88486636259c113955e5176327d3a97ce2ef0a7", size = 43196 }, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -684,6 +774,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/ca/086311cdfc017ec964b2436fe0c98c1f4efcb7e4c328956a22456e497655/fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a", size = 23543 }, ] +[[package]] +name = "findlibs" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/be/6c72ef9d990cd25fe3dd97ebe9d77a859f7d27b7273e62ad750846d207ee/findlibs-0.0.5.tar.gz", hash = "sha256:7a801571e999d0ee83f9b92cbb598c21f861ee26ca9dba74cea8958ba4335e7e", size = 6581 } + [[package]] name = "fonttools" version = "4.54.1" @@ -744,18 +840,19 @@ wheels = [ [[package]] name = "geodata-re" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "boto3" }, { name = "bottleneck" }, - { name = "dask" }, + { name = "dask", extra = ["distributed"] }, { name = "geopandas" }, + { name = "h5netcdf" }, { name = "matplotlib" }, { name = "netcdf4" }, { name = "numexpr" }, { name = "numpy" }, { name = "pandas" }, - { name = "progressbar2" }, + { name = "pvlib" }, { name = "pyproj" }, { name = "pyyaml" }, { name = "rasterio" }, @@ -763,13 +860,18 @@ dependencies = [ { name = "rioxarray" }, { name = "scipy" }, { name = "shapely" }, + { name = "timezonefinder" }, { name = "toolz" }, { name = "tqdm" }, { name = "xarray" }, ] [package.optional-dependencies] +accelerate = [ + { name = "numba" }, +] docs = [ + { name = "doc8" }, { name = "myst-nb" }, { name = "sphinx" }, { name = "sphinx-autoapi" }, @@ -777,6 +879,7 @@ docs = [ ] download = [ { name = "cdsapi" }, + { name = "herbie-data" }, ] notebook = [ { name = "notebook" }, @@ -794,18 +897,22 @@ dev = [ requires-dist = [ { name = "boto3", specifier = "==1.26.46" }, { name = "bottleneck", specifier = ">=1.3.6" }, - { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.3" }, - { name = "dask", specifier = ">=2024.9.0" }, + { name = "cdsapi", marker = "extra == 'download'", specifier = ">=0.7.5" }, + { name = "dask", extras = ["distributed"], specifier = ">=2024.9.0" }, + { name = "doc8", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "geopandas", specifier = ">=1.0.1" }, + { name = "h5netcdf", specifier = ">=1.6.1" }, + { name = "herbie-data", marker = "extra == 'download'", specifier = ">=2025.5.0" }, { name = "matplotlib", specifier = "==3.9.2" }, { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, { name = "netcdf4", specifier = ">=1.7.1.post2" }, { name = "notebook", marker = "extra == 'notebook'", specifier = ">=7.2.2" }, + { name = "numba", marker = "extra == 'accelerate'", specifier = ">=0.61.0" }, { name = "numexpr", specifier = "==2.10.1" }, { name = "numpy", specifier = "<2" }, { name = "pandas", specifier = ">=2.2.3" }, - { name = "progressbar2", specifier = ">=4.5.0" }, - { name = "pyproj", specifier = "==3.6.1" }, + { name = "pvlib", specifier = ">=0.12.0" }, + { name = "pyproj", specifier = ">=3.6.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rasterio", specifier = "==1.4.0" }, { name = "requests", specifier = ">=2.32.3" }, @@ -815,10 +922,12 @@ requires-dist = [ { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0.0" }, { name = "sphinx-autoapi", marker = "extra == 'docs'", specifier = "==3.3.2" }, { name = "sphinx-book-theme", marker = "extra == 'docs'", specifier = ">=1.1.3" }, + { name = "timezonefinder", specifier = ">=6.5.9" }, { name = "toolz", specifier = ">=0.12.1" }, { name = "tqdm", specifier = ">=4.66.5" }, { name = "xarray", specifier = ">=2024.9.0" }, ] +provides-extras = ["download", "notebook", "docs", "accelerate"] [package.metadata.requires-dev] dev = [ @@ -905,6 +1014,101 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, ] +[[package]] +name = "h3" +version = "4.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/2a/dac23698122456ce12345e0a1408c23bc389eaffab5eb7689b8e080d9dfb/h3-4.2.2.tar.gz", hash = "sha256:5cc78d546b5c732f480a842e3e436c393fe37fe59b063fe9cb5589206b7c4c7e", size = 167640 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/d8/3a8900058cd463f65a4aa4482ad46f1e813836b18df2bfb15d93aa6dc6d5/h3-4.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:75bae45428b133c3006a3c72646e42856c8800a74e47a818e1bcd242a86e9dae", size = 887980 }, + { url = "https://files.pythonhosted.org/packages/a3/7d/0f50c45edbe36657665a00ef4e5d210d45716f63b360497f4f8677035019/h3-4.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9992eb522636cbcf7c5b5fd32436395bee9423805ee06fd8ad6eca83f4af654", size = 832701 }, + { url = "https://files.pythonhosted.org/packages/0f/6e/6a140e6fb089eb302bf98cf708eb67cea67ffb128ee17b1b11c1cbad4734/h3-4.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7687de29510ece132e59739bd87dcb3fe25783150502685d7d5be266ea4162b5", size = 977189 }, + { url = "https://files.pythonhosted.org/packages/be/1d/84bd08e64d56deeace7de69e24a115faaf2d8c58d0f02d3661258db0369e/h3-4.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51d715d0471bb581bd94f72529c3194e9b91ce30202672d2050161292f774742", size = 1030504 }, + { url = "https://files.pythonhosted.org/packages/e6/61/602ec16ef539f7285a6fab99beb2f15903f8598e3659d478003979b7ce7a/h3-4.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f4eab1fdfc15b83edac415377790114cae5f117ae7d3293fc9b27354fb2987e6", size = 1078345 }, + { url = "https://files.pythonhosted.org/packages/85/22/5bd067d062fdd43c1093fbe8dbd869c9e6cc80277501a25461c0459cbf51/h3-4.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:6c2677a20c46148a31838e5b4eed11554fd83a9f07c63a2056f4187cb1ba605f", size = 807675 }, + { url = "https://files.pythonhosted.org/packages/af/b7/ef768dad3f93ccb33823c8e9ef3cab3fc23f863f5536ac93e3462d9d79ea/h3-4.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b4039d3bc8236ea371402c755867c6b0d36d072dcd1117e22a9040dd63ba522c", size = 888300 }, + { url = "https://files.pythonhosted.org/packages/cb/51/8aba727a3b453ae62b5a47cfba8cb4dc1c29f209375ed694ef0fe030c855/h3-4.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d637189edf2f1523d625e03415ce4b7fdfe224e6254b73c987b8774214a09edf", size = 832835 }, + { url = "https://files.pythonhosted.org/packages/01/e0/2b2b484416d047f932b3d01cc84f0e8b187abece5ab9044a25c1fa76bfb6/h3-4.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5b88187fcfc88a4315633b0cf27fa55b736948b78833045d03a08a33db4feb3", size = 977615 }, + { url = "https://files.pythonhosted.org/packages/ff/c0/f7986dce65bfdf6812a2fe85aa73f636a9c1ce547cdc481b7b7110abbfba/h3-4.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2459e482147a37292e9638f08bf3b4f32df972457064721b888883c1883848c9", size = 1031287 }, + { url = "https://files.pythonhosted.org/packages/7d/03/d4519bde23b1457fe32d4063e3cc8b95fa61b6abd4c322dcdaafe47421be/h3-4.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4db576b1d96d3d828f1b06eef943e1ad00eb9ff2caae5fb894ec3f4badcfde34", size = 1078669 }, + { url = "https://files.pythonhosted.org/packages/eb/0d/00d8d9b688793f09976cae6e731e7c32641af9d010dec79410f5da13db5c/h3-4.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:db1dc702760a8b94abbd1ddb6efd1e88c63032f0bbd7288159af54ce6a611f3a", size = 808885 }, + { url = "https://files.pythonhosted.org/packages/57/29/853abc62ca5526163b29ec8be0b1229f87d99ac7e7565be738856eda3091/h3-4.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c09b7df785cf8a191e33e3b15ba34aef864b78061b2aeaee87a434aa703bb29c", size = 899353 }, + { url = "https://files.pythonhosted.org/packages/e3/2e/0ade7631f0bb959c72c6eaab05f4d4f009224d0062950975dbf7d5f0db38/h3-4.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:35afe80a011abf9ffa8bc885b847a01a0909fca6556f6f997d6adf7beec87e38", size = 837002 }, + { url = "https://files.pythonhosted.org/packages/33/e9/b75c1cdc11b68064e82720f66d22a97e8296aa59194d132b6cbf09c9fd34/h3-4.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a56aa550e56de5fc9726fd20f484b23a336d6b0844e40604500ce9c3bc8f86c", size = 959920 }, + { url = "https://files.pythonhosted.org/packages/fd/b0/2fb42bd02f5d11baba3e81ed524ec1c4ab92ca73050316f2d2955ecf9c1d/h3-4.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e83d7484441f42a6bcd0ffc9626387e4ff8a7563ae659667f5b59b814fe198b7", size = 1014727 }, + { url = "https://files.pythonhosted.org/packages/5c/10/4a29b577f27e94f087733f3ebffa918e1e66ef22e0d1862ef52f4c9aa040/h3-4.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:737d7cf3aeebc60b0e091220a1bdf94608f16457589cf67acc288444ee5f5dc8", size = 1061560 }, + { url = "https://files.pythonhosted.org/packages/25/15/717815d4b34d2776241de2815ab415aa86a097abf6c76c6b5b67f5954927/h3-4.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bc61382bceff776f0d6278e6ba54d94c9e15137d527adbd50914c7591f9c4460", size = 813445 }, + { url = "https://files.pythonhosted.org/packages/4f/bb/e362d88633f309633f28d958ec0eb813166711972d4538afd337d151169e/h3-4.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d70059de58956d58a92213a0b7adbc4fcf643a102911bc1ad637c99554765add", size = 891287 }, + { url = "https://files.pythonhosted.org/packages/a7/99/0827284f7a8dfe4b7680d5c2143a16d49fa2d2e01a26e6af08b992d96003/h3-4.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a9b32da65d9ed6f97ca9ff404721c97f0faec5656e205a7601628ae402f2b282", size = 829418 }, + { url = "https://files.pythonhosted.org/packages/0d/33/67ffd531b1b9b7f4b01da7f7fcad29b367909146bf3ebc290c3267c02580/h3-4.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bea5b734343c5074c60a575cb609157f51aa1c8da5c4064491bb5c58a4dac55", size = 954805 }, + { url = "https://files.pythonhosted.org/packages/0d/1b/514cfd2dffdf18259b39dc0178a10cb15257ef5365bcbb99b2c682f96c89/h3-4.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7836e8e32c30f34fba1c26cdce8e4e7b4a9f48d18f3a2ab9edfeee995deb92a", size = 1008989 }, + { url = "https://files.pythonhosted.org/packages/59/5d/db71fc5e8d82aa63717d3c12ad9e482b128f66a76f11c17f9c80701f12a8/h3-4.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:652299d3bef76af65f5166bc4af0598afa6c7d93cfc13c7a645a95d8c8e5b10e", size = 1058255 }, + { url = "https://files.pythonhosted.org/packages/b8/c5/6088ff388e73376fc672752059df37bf17b13b3f3b8ed55d178f2833e89c/h3-4.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfda0f2864462f074cc22df75a50b9db010988e341a1871993675bb21954891b", size = 810071 }, +] + +[[package]] +name = "h5netcdf" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h5py" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/6d/c1b8e48fccbb588c23033bf219a3190a50813857d78a4c1aae2e1f3969e9/h5netcdf-1.6.1.tar.gz", hash = "sha256:7ef4ecd811374d94d29ac5e7f7db71ff59b55ef8eeefbe4ccc2c316853d31894", size = 64456 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/fc/e73747f3dd31906bfbb78c76069f67d91525fefa28492a1f949cbb4a3c7f/h5netcdf-1.6.1-py3-none-any.whl", hash = "sha256:1ec75cabd6ab50c6e7109d0c6595eb2960ba0e79fef2257607ab80838d84e6f6", size = 49561 }, +] + +[[package]] +name = "h5py" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/2e/a22d6a8bfa6f8be33e7febd985680fba531562795f0a9077ed1eb047bfb0/h5py-3.13.0.tar.gz", hash = "sha256:1870e46518720023da85d0895a1960ff2ce398c5671eac3b1a41ec696b7105c3", size = 414876 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/8a/bc76588ff1a254e939ce48f30655a8f79fac614ca8bd1eda1a79fa276671/h5py-3.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5540daee2b236d9569c950b417f13fd112d51d78b4c43012de05774908dff3f5", size = 3413286 }, + { url = "https://files.pythonhosted.org/packages/19/bd/9f249ecc6c517b2796330b0aab7d2351a108fdbd00d4bb847c0877b5533e/h5py-3.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:10894c55d46df502d82a7a4ed38f9c3fdbcb93efb42e25d275193e093071fade", size = 2915673 }, + { url = "https://files.pythonhosted.org/packages/72/71/0dd079208d7d3c3988cebc0776c2de58b4d51d8eeb6eab871330133dfee6/h5py-3.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb267ce4b83f9c42560e9ff4d30f60f7ae492eacf9c7ede849edf8c1b860e16b", size = 4283822 }, + { url = "https://files.pythonhosted.org/packages/d8/fa/0b6a59a1043c53d5d287effa02303bd248905ee82b25143c7caad8b340ad/h5py-3.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2cf6a231a07c14acd504a945a6e9ec115e0007f675bde5e0de30a4dc8d86a31", size = 4548100 }, + { url = "https://files.pythonhosted.org/packages/12/42/ad555a7ff7836c943fe97009405566dc77bcd2a17816227c10bd067a3ee1/h5py-3.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:851ae3a8563d87a5a0dc49c2e2529c75b8842582ccaefbf84297d2cfceeacd61", size = 2950547 }, + { url = "https://files.pythonhosted.org/packages/86/2b/50b15fdefb577d073b49699e6ea6a0a77a3a1016c2b67e2149fc50124a10/h5py-3.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8a8e38ef4ceb969f832cc230c0cf808c613cc47e31e768fd7b1106c55afa1cb8", size = 3422922 }, + { url = "https://files.pythonhosted.org/packages/94/59/36d87a559cab9c59b59088d52e86008d27a9602ce3afc9d3b51823014bf3/h5py-3.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f35640e81b03c02a88b8bf99fb6a9d3023cc52f7c627694db2f379e0028f2868", size = 2921619 }, + { url = "https://files.pythonhosted.org/packages/37/ef/6f80b19682c0b0835bbee7b253bec9c16af9004f2fd6427b1dd858100273/h5py-3.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:337af114616f3656da0c83b68fcf53ecd9ce9989a700b0883a6e7c483c3235d4", size = 4259366 }, + { url = "https://files.pythonhosted.org/packages/03/71/c99f662d4832c8835453cf3476f95daa28372023bda4aa1fca9e97c24f09/h5py-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:782ff0ac39f455f21fd1c8ebc007328f65f43d56718a89327eec76677ebf238a", size = 4509058 }, + { url = "https://files.pythonhosted.org/packages/56/89/e3ff23e07131ff73a72a349be9639e4de84e163af89c1c218b939459a98a/h5py-3.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:22ffe2a25770a2d67213a1b94f58006c14dce06933a42d2aaa0318c5868d1508", size = 2966428 }, + { url = "https://files.pythonhosted.org/packages/d8/20/438f6366ba4ded80eadb38f8927f5e2cd6d2e087179552f20ae3dbcd5d5b/h5py-3.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:477c58307b6b9a2509c59c57811afb9f598aedede24a67da808262dfa0ee37b4", size = 3384442 }, + { url = "https://files.pythonhosted.org/packages/10/13/cc1cb7231399617d9951233eb12fddd396ff5d4f7f057ee5d2b1ca0ee7e7/h5py-3.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:57c4c74f627c616f02b7aec608a8c706fe08cb5b0ba7c08555a4eb1dde20805a", size = 2917567 }, + { url = "https://files.pythonhosted.org/packages/9e/d9/aed99e1c858dc698489f916eeb7c07513bc864885d28ab3689d572ba0ea0/h5py-3.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:357e6dc20b101a805ccfd0024731fbaf6e8718c18c09baf3b5e4e9d198d13fca", size = 4669544 }, + { url = "https://files.pythonhosted.org/packages/a7/da/3c137006ff5f0433f0fb076b1ebe4a7bf7b5ee1e8811b5486af98b500dd5/h5py-3.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6f13f9b5ce549448c01e4dfe08ea8d1772e6078799af2c1c8d09e941230a90d", size = 4932139 }, + { url = "https://files.pythonhosted.org/packages/25/61/d897952629cae131c19d4c41b2521e7dd6382f2d7177c87615c2e6dced1a/h5py-3.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:21daf38171753899b5905f3d82c99b0b1ec2cbbe282a037cad431feb620e62ec", size = 2954179 }, + { url = "https://files.pythonhosted.org/packages/60/43/f276f27921919a9144074320ce4ca40882fc67b3cfee81c3f5c7df083e97/h5py-3.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e520ec76de00943dd017c8ea3f354fa1d2f542eac994811943a8faedf2a7d5cb", size = 3358040 }, + { url = "https://files.pythonhosted.org/packages/1b/86/ad4a4cf781b08d4572be8bbdd8f108bb97b266a14835c640dc43dafc0729/h5py-3.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e79d8368cd9295045956bfb436656bea3f915beaa11d342e9f79f129f5178763", size = 2892766 }, + { url = "https://files.pythonhosted.org/packages/69/84/4c6367d6b58deaf0fa84999ec819e7578eee96cea6cbd613640d0625ed5e/h5py-3.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56dd172d862e850823c4af02dc4ddbc308f042b85472ffdaca67f1598dff4a57", size = 4664255 }, + { url = "https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be949b46b7388074c5acae017fbbe3e5ba303fd9daaa52157fdfef30bbdacadd", size = 4927580 }, + { url = "https://files.pythonhosted.org/packages/97/34/165b87ea55184770a0c1fcdb7e017199974ad2e271451fd045cfe35f3add/h5py-3.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:4f97ecde7ac6513b21cd95efdfc38dc6d19f96f6ca6f2a30550e94e551458e0a", size = 2940890 }, +] + +[[package]] +name = "herbie-data" +version = "2025.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgrib" }, + { name = "eccodes" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pyproj" }, + { name = "requests" }, + { name = "toml" }, + { name = "xarray" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/91/cb0e1effccf51daffb615393d114a8110a466c15d41820a1d242294a2481/herbie_data-2025.5.0.tar.gz", hash = "sha256:0629374ae29d0ece5f58c9369219ed95ec262ac0544d6ecb7824e79fdc3a2aef", size = 9418650 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/47/8f8a5af62c941c5b7e415a8735d5c81f51943b0463b91974e269317e9038/herbie_data-2025.5.0-py3-none-any.whl", hash = "sha256:57eb86b456d626cd057aca42a4b1c3c4bf11a3c10ae0696e47e52891c82b4263", size = 113296 }, +] + [[package]] name = "httpcore" version = "1.0.6" @@ -978,7 +1182,7 @@ name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "platform_system == 'Darwin'" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, { name = "comm" }, { name = "debugpy" }, { name = "ipython" }, @@ -1382,6 +1586,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/8b/d7497df4a1cae9367adf21665dd1f896c2a7aeb8769ad77b662c5e2bcce7/kiwisolver-1.4.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:eb542fe7933aa09d8d8f9d9097ef37532a7df6497819d16efe4359890a2f417a", size = 55715 }, ] +[[package]] +name = "llvmlite" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/6a/95a3d3610d5c75293d5dbbb2a76480d5d4eeba641557b69fe90af6c5b84e/llvmlite-0.44.0.tar.gz", hash = "sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4", size = 171880 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/75/d4863ddfd8ab5f6e70f4504cf8cc37f4e986ec6910f4ef8502bb7d3c1c71/llvmlite-0.44.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9fbadbfba8422123bab5535b293da1cf72f9f478a65645ecd73e781f962ca614", size = 28132306 }, + { url = "https://files.pythonhosted.org/packages/37/d9/6e8943e1515d2f1003e8278819ec03e4e653e2eeb71e4d00de6cfe59424e/llvmlite-0.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cccf8eb28f24840f2689fb1a45f9c0f7e582dd24e088dcf96e424834af11f791", size = 26201096 }, + { url = "https://files.pythonhosted.org/packages/aa/46/8ffbc114def88cc698906bf5acab54ca9fdf9214fe04aed0e71731fb3688/llvmlite-0.44.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7202b678cdf904823c764ee0fe2dfe38a76981f4c1e51715b4cb5abb6cf1d9e8", size = 42361859 }, + { url = "https://files.pythonhosted.org/packages/30/1c/9366b29ab050a726af13ebaae8d0dff00c3c58562261c79c635ad4f5eb71/llvmlite-0.44.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40526fb5e313d7b96bda4cbb2c85cd5374e04d80732dd36a282d72a560bb6408", size = 41184199 }, + { url = "https://files.pythonhosted.org/packages/69/07/35e7c594b021ecb1938540f5bce543ddd8713cff97f71d81f021221edc1b/llvmlite-0.44.0-cp310-cp310-win_amd64.whl", hash = "sha256:41e3839150db4330e1b2716c0be3b5c4672525b4c9005e17c7597f835f351ce2", size = 30332381 }, + { url = "https://files.pythonhosted.org/packages/b5/e2/86b245397052386595ad726f9742e5223d7aea999b18c518a50e96c3aca4/llvmlite-0.44.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3", size = 28132305 }, + { url = "https://files.pythonhosted.org/packages/ff/ec/506902dc6870249fbe2466d9cf66d531265d0f3a1157213c8f986250c033/llvmlite-0.44.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ace564d9fa44bb91eb6e6d8e7754977783c68e90a471ea7ce913bff30bd62427", size = 26201090 }, + { url = "https://files.pythonhosted.org/packages/99/fe/d030f1849ebb1f394bb3f7adad5e729b634fb100515594aca25c354ffc62/llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5d22c3bfc842668168a786af4205ec8e3ad29fb1bc03fd11fd48460d0df64c1", size = 42361858 }, + { url = "https://files.pythonhosted.org/packages/d7/7a/ce6174664b9077fc673d172e4c888cb0b128e707e306bc33fff8c2035f0d/llvmlite-0.44.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610", size = 41184200 }, + { url = "https://files.pythonhosted.org/packages/5f/c6/258801143975a6d09a373f2641237992496e15567b907a4d401839d671b8/llvmlite-0.44.0-cp311-cp311-win_amd64.whl", hash = "sha256:d8489634d43c20cd0ad71330dde1d5bc7b9966937a263ff1ec1cebb90dc50955", size = 30331193 }, + { url = "https://files.pythonhosted.org/packages/15/86/e3c3195b92e6e492458f16d233e58a1a812aa2bfbef9bdd0fbafcec85c60/llvmlite-0.44.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:1d671a56acf725bf1b531d5ef76b86660a5ab8ef19bb6a46064a705c6ca80aad", size = 28132297 }, + { url = "https://files.pythonhosted.org/packages/d6/53/373b6b8be67b9221d12b24125fd0ec56b1078b660eeae266ec388a6ac9a0/llvmlite-0.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f79a728e0435493611c9f405168682bb75ffd1fbe6fc360733b850c80a026db", size = 26201105 }, + { url = "https://files.pythonhosted.org/packages/cb/da/8341fd3056419441286c8e26bf436923021005ece0bff5f41906476ae514/llvmlite-0.44.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0143a5ef336da14deaa8ec26c5449ad5b6a2b564df82fcef4be040b9cacfea9", size = 42361901 }, + { url = "https://files.pythonhosted.org/packages/53/ad/d79349dc07b8a395a99153d7ce8b01d6fcdc9f8231355a5df55ded649b61/llvmlite-0.44.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d752f89e31b66db6f8da06df8b39f9b91e78c5feea1bf9e8c1fba1d1c24c065d", size = 41184247 }, + { url = "https://files.pythonhosted.org/packages/e2/3b/a9a17366af80127bd09decbe2a54d8974b6d8b274b39bf47fbaedeec6307/llvmlite-0.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:eae7e2d4ca8f88f89d315b48c6b741dcb925d6a1042da694aa16ab3dd4cbd3a1", size = 30332380 }, + { url = "https://files.pythonhosted.org/packages/89/24/4c0ca705a717514c2092b18476e7a12c74d34d875e05e4d742618ebbf449/llvmlite-0.44.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:319bddd44e5f71ae2689859b7203080716448a3cd1128fb144fe5c055219d516", size = 28132306 }, + { url = "https://files.pythonhosted.org/packages/01/cf/1dd5a60ba6aee7122ab9243fd614abcf22f36b0437cbbe1ccf1e3391461c/llvmlite-0.44.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c58867118bad04a0bb22a2e0068c693719658105e40009ffe95c7000fcde88e", size = 26201090 }, + { url = "https://files.pythonhosted.org/packages/d2/1b/656f5a357de7135a3777bd735cc7c9b8f23b4d37465505bd0eaf4be9befe/llvmlite-0.44.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46224058b13c96af1365290bdfebe9a6264ae62fb79b2b55693deed11657a8bf", size = 42361904 }, + { url = "https://files.pythonhosted.org/packages/d8/e1/12c5f20cb9168fb3464a34310411d5ad86e4163c8ff2d14a2b57e5cc6bac/llvmlite-0.44.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0097052c32bf721a4efc03bd109d335dfa57d9bffb3d4c24cc680711b8b4fc", size = 41184245 }, + { url = "https://files.pythonhosted.org/packages/d0/81/e66fc86539293282fd9cb7c9417438e897f369e79ffb62e1ae5e5154d4dd/llvmlite-0.44.0-cp313-cp313-win_amd64.whl", hash = "sha256:2fb7c4f2fb86cbae6dca3db9ab203eeea0e22d73b99bc2341cdf9de93612e930", size = 30331193 }, +] + [[package]] name = "locket" version = "1.0.0" @@ -1531,9 +1763,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/74/c95adcdf032956d9ef6c89a9b8a5152bf73915f8c633f3e3d88d06bd699c/mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205", size = 47958 }, ] +[[package]] +name = "msgpack" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d0/7555686ae7ff5731205df1012ede15dd9d927f6227ea151e901c7406af4f/msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e", size = 167260 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/f9/a892a6038c861fa849b11a2bb0502c07bc698ab6ea53359e5771397d883b/msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd", size = 150428 }, + { url = "https://files.pythonhosted.org/packages/df/7a/d174cc6a3b6bb85556e6a046d3193294a92f9a8e583cdbd46dc8a1d7e7f4/msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d", size = 84131 }, + { url = "https://files.pythonhosted.org/packages/08/52/bf4fbf72f897a23a56b822997a72c16de07d8d56d7bf273242f884055682/msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5", size = 81215 }, + { url = "https://files.pythonhosted.org/packages/02/95/dc0044b439b518236aaf012da4677c1b8183ce388411ad1b1e63c32d8979/msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5", size = 371229 }, + { url = "https://files.pythonhosted.org/packages/ff/75/09081792db60470bef19d9c2be89f024d366b1e1973c197bb59e6aabc647/msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e", size = 378034 }, + { url = "https://files.pythonhosted.org/packages/32/d3/c152e0c55fead87dd948d4b29879b0f14feeeec92ef1fd2ec21b107c3f49/msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b", size = 363070 }, + { url = "https://files.pythonhosted.org/packages/d9/2c/82e73506dd55f9e43ac8aa007c9dd088c6f0de2aa19e8f7330e6a65879fc/msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f", size = 359863 }, + { url = "https://files.pythonhosted.org/packages/cb/a0/3d093b248837094220e1edc9ec4337de3443b1cfeeb6e0896af8ccc4cc7a/msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68", size = 368166 }, + { url = "https://files.pythonhosted.org/packages/e4/13/7646f14f06838b406cf5a6ddbb7e8dc78b4996d891ab3b93c33d1ccc8678/msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b", size = 370105 }, + { url = "https://files.pythonhosted.org/packages/67/fa/dbbd2443e4578e165192dabbc6a22c0812cda2649261b1264ff515f19f15/msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044", size = 68513 }, + { url = "https://files.pythonhosted.org/packages/24/ce/c2c8fbf0ded750cb63cbcbb61bc1f2dfd69e16dca30a8af8ba80ec182dcd/msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f", size = 74687 }, + { url = "https://files.pythonhosted.org/packages/b7/5e/a4c7154ba65d93be91f2f1e55f90e76c5f91ccadc7efc4341e6f04c8647f/msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7", size = 150803 }, + { url = "https://files.pythonhosted.org/packages/60/c2/687684164698f1d51c41778c838d854965dd284a4b9d3a44beba9265c931/msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa", size = 84343 }, + { url = "https://files.pythonhosted.org/packages/42/ae/d3adea9bb4a1342763556078b5765e666f8fdf242e00f3f6657380920972/msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701", size = 81408 }, + { url = "https://files.pythonhosted.org/packages/dc/17/6313325a6ff40ce9c3207293aee3ba50104aed6c2c1559d20d09e5c1ff54/msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6", size = 396096 }, + { url = "https://files.pythonhosted.org/packages/a8/a1/ad7b84b91ab5a324e707f4c9761633e357820b011a01e34ce658c1dda7cc/msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59", size = 403671 }, + { url = "https://files.pythonhosted.org/packages/bb/0b/fd5b7c0b308bbf1831df0ca04ec76fe2f5bf6319833646b0a4bd5e9dc76d/msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0", size = 387414 }, + { url = "https://files.pythonhosted.org/packages/f0/03/ff8233b7c6e9929a1f5da3c7860eccd847e2523ca2de0d8ef4878d354cfa/msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e", size = 383759 }, + { url = "https://files.pythonhosted.org/packages/1f/1b/eb82e1fed5a16dddd9bc75f0854b6e2fe86c0259c4353666d7fab37d39f4/msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6", size = 394405 }, + { url = "https://files.pythonhosted.org/packages/90/2e/962c6004e373d54ecf33d695fb1402f99b51832631e37c49273cc564ffc5/msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5", size = 396041 }, + { url = "https://files.pythonhosted.org/packages/f8/20/6e03342f629474414860c48aeffcc2f7f50ddaf351d95f20c3f1c67399a8/msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88", size = 68538 }, + { url = "https://files.pythonhosted.org/packages/aa/c4/5a582fc9a87991a3e6f6800e9bb2f3c82972912235eb9539954f3e9997c7/msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788", size = 74871 }, + { url = "https://files.pythonhosted.org/packages/e1/d6/716b7ca1dbde63290d2973d22bbef1b5032ca634c3ff4384a958ec3f093a/msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d", size = 152421 }, + { url = "https://files.pythonhosted.org/packages/70/da/5312b067f6773429cec2f8f08b021c06af416bba340c912c2ec778539ed6/msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2", size = 85277 }, + { url = "https://files.pythonhosted.org/packages/28/51/da7f3ae4462e8bb98af0d5bdf2707f1b8c65a0d4f496e46b6afb06cbc286/msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420", size = 82222 }, + { url = "https://files.pythonhosted.org/packages/33/af/dc95c4b2a49cff17ce47611ca9ba218198806cad7796c0b01d1e332c86bb/msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2", size = 392971 }, + { url = "https://files.pythonhosted.org/packages/f1/54/65af8de681fa8255402c80eda2a501ba467921d5a7a028c9c22a2c2eedb5/msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39", size = 401403 }, + { url = "https://files.pythonhosted.org/packages/97/8c/e333690777bd33919ab7024269dc3c41c76ef5137b211d776fbb404bfead/msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f", size = 385356 }, + { url = "https://files.pythonhosted.org/packages/57/52/406795ba478dc1c890559dd4e89280fa86506608a28ccf3a72fbf45df9f5/msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247", size = 383028 }, + { url = "https://files.pythonhosted.org/packages/e7/69/053b6549bf90a3acadcd8232eae03e2fefc87f066a5b9fbb37e2e608859f/msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c", size = 391100 }, + { url = "https://files.pythonhosted.org/packages/23/f0/d4101d4da054f04274995ddc4086c2715d9b93111eb9ed49686c0f7ccc8a/msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b", size = 394254 }, + { url = "https://files.pythonhosted.org/packages/1c/12/cf07458f35d0d775ff3a2dc5559fa2e1fcd06c46f1ef510e594ebefdca01/msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b", size = 69085 }, + { url = "https://files.pythonhosted.org/packages/73/80/2708a4641f7d553a63bc934a3eb7214806b5b39d200133ca7f7afb0a53e8/msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f", size = 75347 }, + { url = "https://files.pythonhosted.org/packages/c8/b0/380f5f639543a4ac413e969109978feb1f3c66e931068f91ab6ab0f8be00/msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf", size = 151142 }, + { url = "https://files.pythonhosted.org/packages/c8/ee/be57e9702400a6cb2606883d55b05784fada898dfc7fd12608ab1fdb054e/msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330", size = 84523 }, + { url = "https://files.pythonhosted.org/packages/7e/3a/2919f63acca3c119565449681ad08a2f84b2171ddfcff1dba6959db2cceb/msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734", size = 81556 }, + { url = "https://files.pythonhosted.org/packages/7c/43/a11113d9e5c1498c145a8925768ea2d5fce7cbab15c99cda655aa09947ed/msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e", size = 392105 }, + { url = "https://files.pythonhosted.org/packages/2d/7b/2c1d74ca6c94f70a1add74a8393a0138172207dc5de6fc6269483519d048/msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca", size = 399979 }, + { url = "https://files.pythonhosted.org/packages/82/8c/cf64ae518c7b8efc763ca1f1348a96f0e37150061e777a8ea5430b413a74/msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915", size = 383816 }, + { url = "https://files.pythonhosted.org/packages/69/86/a847ef7a0f5ef3fa94ae20f52a4cacf596a4e4a010197fbcc27744eb9a83/msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d", size = 380973 }, + { url = "https://files.pythonhosted.org/packages/aa/90/c74cf6e1126faa93185d3b830ee97246ecc4fe12cf9d2d31318ee4246994/msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434", size = 387435 }, + { url = "https://files.pythonhosted.org/packages/7a/40/631c238f1f338eb09f4acb0f34ab5862c4e9d7eda11c1b685471a4c5ea37/msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c", size = 399082 }, + { url = "https://files.pythonhosted.org/packages/e9/1b/fa8a952be252a1555ed39f97c06778e3aeb9123aa4cccc0fd2acd0b4e315/msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc", size = 69037 }, + { url = "https://files.pythonhosted.org/packages/b6/bc/8bd826dd03e022153bfa1766dcdec4976d6c818865ed54223d71f07862b3/msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f", size = 75140 }, +] + [[package]] name = "multiurl" -version = "0.3.1" +version = "0.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, @@ -1541,7 +1825,7 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cc/12/4e979f71d90ca5625647f93cd484c733a7e8ae4fd9f6d15369613d727301/multiurl-0.3.1.tar.gz", hash = "sha256:c7001437b59d56d4c310d725c3dcfff98c97c4b652893d88989853827465d442", size = 18161 } +sdist = { url = "https://files.pythonhosted.org/packages/33/db/aad981174d3bdaecc1d7e1f2d176641f022300ddf3a17b13c2775d041b7a/multiurl-0.3.3.tar.gz", hash = "sha256:f4d0b69dcf4a0ed740daa313dbcd4d5665420d305c50ca879285e96dc828093f", size = 18382 } [[package]] name = "mypy-extensions" @@ -1711,6 +1995,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307 }, ] +[[package]] +name = "numba" +version = "0.61.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/88/c13a935f200fda51384411e49840a8e7f70c9cb1ee8d809dd0f2477cf7ef/numba-0.61.0.tar.gz", hash = "sha256:888d2e89b8160899e19591467e8fdd4970e07606e1fbc248f239c89818d5f925", size = 2816484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/97/8568a025b9ab8b4d53491e70d4206d5f3fc71fbe94f3097058e01ad8e7ff/numba-0.61.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9cab9783a700fa428b1a54d65295122bc03b3de1d01fb819a6b9dbbddfdb8c43", size = 2769008 }, + { url = "https://files.pythonhosted.org/packages/8c/ab/a88c20755f66543ee01c85c98b866595b92e1bd0ed80565a4889e22929a8/numba-0.61.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:46c5ae094fb3706f5adf9021bfb7fc11e44818d61afee695cdee4eadfed45e98", size = 2771815 }, + { url = "https://files.pythonhosted.org/packages/ae/f4/b357913089ecec1a9ddc6adc04090396928f36a484a5ab9e71b24ddba4cd/numba-0.61.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6fb74e81aa78a2303e30593d8331327dfc0d2522b5db05ac967556a26db3ef87", size = 3820233 }, + { url = "https://files.pythonhosted.org/packages/ea/60/0e21bcf3baaf10e39d48cd224618e46a6b75d3394f465c37ce57bf98cbfa/numba-0.61.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ebbd4827091384ab8c4615ba1b3ca8bc639a3a000157d9c37ba85d34cd0da1b", size = 3514707 }, + { url = "https://files.pythonhosted.org/packages/a0/08/45c136ab59e6b11e61ce15a0d17ef03fd89eaccb0db05ad67912aaf5218a/numba-0.61.0-cp310-cp310-win_amd64.whl", hash = "sha256:43aa4d7d10c542d3c78106b8481e0cbaaec788c39ee8e3d7901682748ffdf0b4", size = 2827753 }, + { url = "https://files.pythonhosted.org/packages/63/8f/f983a7c859ccad73d3cc3f86fbba94f16e137cd1ee464631d61b624363b2/numba-0.61.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:bf64c2d0f3d161af603de3825172fb83c2600bcb1d53ae8ea568d4c53ba6ac08", size = 2768960 }, + { url = "https://files.pythonhosted.org/packages/be/1b/c33dc847d475d5b647b4ad5aefc38df7a72283763f4cda47745050375a81/numba-0.61.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de5aa7904741425f28e1028b85850b31f0a245e9eb4f7c38507fb893283a066c", size = 2771862 }, + { url = "https://files.pythonhosted.org/packages/14/91/18b9f64b34ff318a14d072251480547f89ebfb864b2b7168e5dc5f64f502/numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21c2fe25019267a608e2710a6a947f557486b4b0478b02e45a81cf606a05a7d4", size = 3825411 }, + { url = "https://files.pythonhosted.org/packages/f2/97/1a38030c2a331e273ace1de2b61988e33d80878fda8a5eedee0cd78399d3/numba-0.61.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:74250b26ed6a1428763e774dc5b2d4e70d93f73795635b5412b8346a4d054574", size = 3519604 }, + { url = "https://files.pythonhosted.org/packages/df/a7/56f547de8fc197963f238fd62beb5f1d2cace047602d0577956bf6840970/numba-0.61.0-cp311-cp311-win_amd64.whl", hash = "sha256:b72bbc8708e98b3741ad0c63f9929c47b623cc4ee86e17030a4f3e301e8401ac", size = 2827642 }, + { url = "https://files.pythonhosted.org/packages/63/c9/c61881e7f2e253e745209f078bbd428ce23b6cf901f7d93afe166720ff95/numba-0.61.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:152146ecdbb8d8176f294e9f755411e6f270103a11c3ff50cecc413f794e52c8", size = 2769758 }, + { url = "https://files.pythonhosted.org/packages/e1/28/ddec0147a4933f86ceaca580aa9bb767d5632ecdb1ece6cfb3eab4ac78e5/numba-0.61.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5cafa6095716fcb081618c28a8d27bf7c001e09696f595b41836dec114be2905", size = 2772445 }, + { url = "https://files.pythonhosted.org/packages/18/74/6a9f0e6c76c088f8a6aa702eab31734068061dca5cc0f34e8bc1eb447de1/numba-0.61.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ffe9fe373ed30638d6e20a0269f817b2c75d447141f55a675bfcf2d1fe2e87fb", size = 3882115 }, + { url = "https://files.pythonhosted.org/packages/53/68/d7c31e53f08e6b4669c9b5a3cd7c5fb9097220c5ef388bc099ca8ab9749f/numba-0.61.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9f25f7fef0206d55c1cfb796ad833cbbc044e2884751e56e798351280038484c", size = 3573296 }, + { url = "https://files.pythonhosted.org/packages/94/4f/8357a99a14f331b865a42cb4756ae37da85599b9c95e01277ea10361e91a/numba-0.61.0-cp312-cp312-win_amd64.whl", hash = "sha256:550d389573bc3b895e1ccb18289feea11d937011de4d278b09dc7ed585d1cdcb", size = 2828077 }, + { url = "https://files.pythonhosted.org/packages/3b/54/71fba18e4af5619f1ea8175ee92e82dd8e220bd6feb8c0153c6b814c8a60/numba-0.61.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:b96fafbdcf6f69b69855273e988696aae4974115a815f6818fef4af7afa1f6b8", size = 2768024 }, + { url = "https://files.pythonhosted.org/packages/39/76/2448b43d08e904aad1b1b9cd12835b19411e84a81aa9192f83642a5e0afd/numba-0.61.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f6c452dca1de8e60e593f7066df052dd8da09b243566ecd26d2b796e5d3087d", size = 2769541 }, + { url = "https://files.pythonhosted.org/packages/32/8f/4bb2374247ab988c9eac587b304b2947a36d605b9bb9ba4bf06e955c17d3/numba-0.61.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44240e694d4aa321430c97b21453e46014fe6c7b8b7d932afa7f6a88cc5d7e5e", size = 3890102 }, + { url = "https://files.pythonhosted.org/packages/ab/bc/dc2d03555289ae5263f65c01d45eb186ce347585c191daf0e60021d5ed39/numba-0.61.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:764f0e47004f126f58c3b28e0a02374c420a9d15157b90806d68590f5c20cc89", size = 3580239 }, + { url = "https://files.pythonhosted.org/packages/61/08/71247ce560d2c222d9ca705c7d3547fc4069b96fc85d71aabeb890befe9f/numba-0.61.0-cp313-cp313-win_amd64.whl", hash = "sha256:074cd38c5b1f9c65a4319d1f3928165f48975ef0537ad43385b2bd908e6e2e35", size = 2828035 }, +] + [[package]] name = "numexpr" version = "2.10.1" @@ -1881,6 +2197,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, ] +[[package]] +name = "pbr" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/d2/510cc0d218e753ba62a1bc1434651db3cd797a9716a0a66cc714cb4f0935/pbr-6.1.1.tar.gz", hash = "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b", size = 125702 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/ac/684d71315abc7b1214d59304e23a982472967f6bf4bde5a98f1503f648dc/pbr-6.1.1-py2.py3-none-any.whl", hash = "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76", size = 108997 }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -1970,18 +2298,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, ] -[[package]] -name = "progressbar2" -version = "4.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-utils" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/24/3587e795fc590611434e4bcb9fbe0c3dddb5754ce1a20edfd86c587c0004/progressbar2-4.5.0.tar.gz", hash = "sha256:6662cb624886ed31eb94daf61e27583b5144ebc7383a17bae076f8f4f59088fb", size = 101449 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/94/448f037fb0ffd0e8a63b625cf9f5b13494b88d15573a987be8aaa735579d/progressbar2-4.5.0-py3-none-any.whl", hash = "sha256:625c94a54e63915b3959355e6d4aacd63a00219e5f3e2b12181b76867bf6f628", size = 57132 }, -] - [[package]] name = "prometheus-client" version = "0.21.0" @@ -2036,6 +2352,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "pvlib" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h5py" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pytz" }, + { name = "requests" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/54/d7b2f3ee461fecd1ef5b2d24aa32d2e167f260c39c614dbd8cc4285937ce/pvlib-0.12.0.tar.gz", hash = "sha256:222bcb7b88f228944e7f7e9317082b7182232b8d0a5d1b1499cff50027d1e964", size = 35715594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/2b/c3766d6a78ab60ff1d300a1d3479134bdae6073c84bd310fc95912b7516e/pvlib-0.12.0-py3-none-any.whl", hash = "sha256:5922991b6a1ad9c4c90b114b582c8ac91ec4bb2d5fb04ce00eac83581316fa2e", size = 19325114 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -2121,31 +2454,45 @@ wheels = [ [[package]] name = "pyproj" -version = "3.6.1" +version = "3.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/84/2b39bbf888c753ea48b40d47511548c77aa03445465c35cc4c4e9649b643/pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf", size = 225131 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/32/63cf474f4a8d4804b3bdf7c16b8589f38142e8e2f8319dcea27e0bc21a87/pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4", size = 6142763 }, - { url = "https://files.pythonhosted.org/packages/18/86/2e7cb9de40492f1bafbf11f4c9072edc394509a40b5e4c52f8139546f039/pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe", size = 4877123 }, - { url = "https://files.pythonhosted.org/packages/5e/c5/928d5a26995dbefbebd7507d982141cd9153bc7e4392b334fff722c4af12/pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f", size = 6190576 }, - { url = "https://files.pythonhosted.org/packages/f6/2b/b60cf73b0720abca313bfffef34e34f7f7dae23852b2853cf0368d49426b/pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35", size = 8328075 }, - { url = "https://files.pythonhosted.org/packages/d9/a8/7193f46032636be917bc775506ae987aad72c931b1f691b775ca812a2917/pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f", size = 5635713 }, - { url = "https://files.pythonhosted.org/packages/89/8f/27350c8fba71a37cd0d316f100fbd96bf139cc2b5ff1ab0dcbc7ac64010a/pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e", size = 6087932 }, - { url = "https://files.pythonhosted.org/packages/84/a6/a300c1b14b2112e966e9f90b18f9c13b586bdcf417207cee913ae9005da3/pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882", size = 6147442 }, - { url = "https://files.pythonhosted.org/packages/30/bd/b9bd3761f08754e8dbb34c5a647db2099b348ab5da338e90980caf280e37/pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a", size = 4880331 }, - { url = "https://files.pythonhosted.org/packages/f4/0a/d82aeeb605b5d6870bc72307c3b5e044e632eb7720df8885e144f51a8eac/pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85", size = 6192425 }, - { url = "https://files.pythonhosted.org/packages/64/90/dfe5c00de1ca4dbb82606e79790659d4ed7f0ed8d372bccb3baca2a5abe0/pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915", size = 8571478 }, - { url = "https://files.pythonhosted.org/packages/14/6d/ae373629a1723f0db80d7b8c93598b00d9ecb930ed9ebf4f35826a33e97c/pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132", size = 5634575 }, - { url = "https://files.pythonhosted.org/packages/79/95/eb68113c5b5737c342bde1bab92705dabe69c16299c5a122616e50f1fbd6/pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2", size = 6088494 }, - { url = "https://files.pythonhosted.org/packages/0b/64/93232511a7906a492b1b7dfdfc17f4e95982d76a24ef4f86d18cfe7ae2c9/pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746", size = 6135280 }, - { url = "https://files.pythonhosted.org/packages/10/f2/b550b1f65cc7e51c9116b220b50aade60c439103432a3fd5b12efbc77e15/pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8", size = 4880030 }, - { url = "https://files.pythonhosted.org/packages/fe/4b/2f8f6f94643b9fe2083338eff294feda84d916409b5840b7a402d2be93f8/pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf", size = 6184439 }, - { url = "https://files.pythonhosted.org/packages/19/9b/c57569132174786aa3f72275ac306956859a639dad0ce8d95c8411ce8209/pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d", size = 8660747 }, - { url = "https://files.pythonhosted.org/packages/0e/ab/1c2159ec757677c5a6b8803f6be45c2b550dc42c84ec4a228dc219849bbb/pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb", size = 5626805 }, - { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779 }, +sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524 }, + { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102 }, + { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667 }, + { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185 }, + { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103 }, + { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391 }, + { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997 }, + { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581 }, + { url = "https://files.pythonhosted.org/packages/ab/0d/63670fc527e664068b70b7cab599aa38b7420dd009bdc29ea257e7f3dfb3/pyproj-3.7.1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a94e26c1a4950cea40116775588a2ca7cf56f1f434ff54ee35a84718f3841a3d", size = 6264315 }, + { url = "https://files.pythonhosted.org/packages/25/9d/cbaf82cfb290d1f1fa42feb9ba9464013bb3891e40c4199f8072112e4589/pyproj-3.7.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:263b54ba5004b6b957d55757d846fc5081bc02980caa0279c4fc95fa0fff6067", size = 4666267 }, + { url = "https://files.pythonhosted.org/packages/79/53/24f9f9b8918c0550f3ff49ad5de4cf3f0688c9f91ff191476db8979146fe/pyproj-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d6a2ccd5607cd15ef990c51e6f2dd27ec0a741e72069c387088bba3aab60fa", size = 9680510 }, + { url = "https://files.pythonhosted.org/packages/3c/ac/12fab74a908d40b63174dc704587febd0729414804bbfd873cabe504ff2d/pyproj-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5dcf24ede53d8abab7d8a77f69ff1936c6a8843ef4fcc574646e4be66e5739", size = 9493619 }, + { url = "https://files.pythonhosted.org/packages/c4/45/26311d6437135da2153a178125db5dfb6abce831ce04d10ec207eabac70a/pyproj-3.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c2e7449840a44ce860d8bea2c6c1c4bc63fa07cba801dcce581d14dcb031a02", size = 10709755 }, + { url = "https://files.pythonhosted.org/packages/99/52/4ecd0986f27d0e6c8ee3a7bc5c63da15acd30ac23034f871325b297e61fd/pyproj-3.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0829865c1d3a3543f918b3919dc601eea572d6091c0dd175e1a054db9c109274", size = 10642970 }, + { url = "https://files.pythonhosted.org/packages/3f/a5/d3bfc018fc92195a000d1d28acc1f3f1df15ff9f09ece68f45a2636c0134/pyproj-3.7.1-cp311-cp311-win32.whl", hash = "sha256:6181960b4b812e82e588407fe5c9c68ada267c3b084db078f248db5d7f45d18a", size = 5868295 }, + { url = "https://files.pythonhosted.org/packages/92/39/ef6f06a5b223dbea308cfcbb7a0f72e7b506aef1850e061b2c73b0818715/pyproj-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ad0ff443a785d84e2b380869fdd82e6bfc11eba6057d25b4409a9bbfa867970", size = 6279871 }, + { url = "https://files.pythonhosted.org/packages/e6/c9/876d4345b8d17f37ac59ebd39f8fa52fc6a6a9891a420f72d050edb6b899/pyproj-3.7.1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:2781029d90df7f8d431e29562a3f2d8eafdf233c4010d6fc0381858dc7373217", size = 6264087 }, + { url = "https://files.pythonhosted.org/packages/ff/e6/5f8691f8c90e7f402cc80a6276eb19d2ec1faa150d5ae2dd9c7b0a254da8/pyproj-3.7.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d61bf8ab04c73c1da08eedaf21a103b72fa5b0a9b854762905f65ff8b375d394", size = 4669628 }, + { url = "https://files.pythonhosted.org/packages/42/ec/16475bbb79c1c68845c0a0d9c60c4fb31e61b8a2a20bc18b1a81e81c7f68/pyproj-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04abc517a8555d1b05fcee768db3280143fe42ec39fdd926a2feef31631a1f2f", size = 9721415 }, + { url = "https://files.pythonhosted.org/packages/b3/a3/448f05b15e318bd6bea9a32cfaf11e886c4ae61fa3eee6e09ed5c3b74bb2/pyproj-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084c0a475688f934d386c2ab3b6ce03398a473cd48adfda70d9ab8f87f2394a0", size = 9556447 }, + { url = "https://files.pythonhosted.org/packages/6a/ae/bd15fe8d8bd914ead6d60bca7f895a4e6f8ef7e3928295134ff9a7dad14c/pyproj-3.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a20727a23b1e49c7dc7fe3c3df8e56a8a7acdade80ac2f5cca29d7ca5564c145", size = 10758317 }, + { url = "https://files.pythonhosted.org/packages/9d/d9/5ccefb8bca925f44256b188a91c31238cae29ab6ee7f53661ecc04616146/pyproj-3.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bf84d766646f1ebd706d883755df4370aaf02b48187cedaa7e4239f16bc8213d", size = 10771259 }, + { url = "https://files.pythonhosted.org/packages/2a/7d/31dedff9c35fa703162f922eeb0baa6c44a3288469a5fd88d209e2892f9e/pyproj-3.7.1-cp312-cp312-win32.whl", hash = "sha256:5f0da2711364d7cb9f115b52289d4a9b61e8bca0da57f44a3a9d6fc9bdeb7274", size = 5859914 }, + { url = "https://files.pythonhosted.org/packages/3e/47/c6ab03d6564a7c937590cff81a2742b5990f096cce7c1a622d325be340ee/pyproj-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:aee664a9d806612af30a19dba49e55a7a78ebfec3e9d198f6a6176e1d140ec98", size = 6273196 }, + { url = "https://files.pythonhosted.org/packages/ef/01/984828464c9960036c602753fc0f21f24f0aa9043c18fa3f2f2b66a86340/pyproj-3.7.1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:5f8d02ef4431dee414d1753d13fa82a21a2f61494737b5f642ea668d76164d6d", size = 6253062 }, + { url = "https://files.pythonhosted.org/packages/68/65/6ecdcdc829811a2c160cdfe2f068a009fc572fd4349664f758ccb0853a7c/pyproj-3.7.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0b853ae99bda66cbe24b4ccfe26d70601d84375940a47f553413d9df570065e0", size = 4660548 }, + { url = "https://files.pythonhosted.org/packages/67/da/dda94c4490803679230ba4c17a12f151b307a0d58e8110820405ca2d98db/pyproj-3.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83db380c52087f9e9bdd8a527943b2e7324f275881125e39475c4f9277bdeec4", size = 9662464 }, + { url = "https://files.pythonhosted.org/packages/6f/57/f61b7d22c91ae1d12ee00ac4c0038714e774ebcd851b9133e5f4f930dd40/pyproj-3.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b35ed213892e211a3ce2bea002aa1183e1a2a9b79e51bb3c6b15549a831ae528", size = 9497461 }, + { url = "https://files.pythonhosted.org/packages/b7/f6/932128236f79d2ac7d39fe1a19667fdf7155d9a81d31fb9472a7a497790f/pyproj-3.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8b15b0463d1303bab113d1a6af2860a0d79013c3a66fcc5475ce26ef717fd4f", size = 10708869 }, + { url = "https://files.pythonhosted.org/packages/1d/0d/07ac7712994454a254c383c0d08aff9916a2851e6512d59da8dc369b1b02/pyproj-3.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:87229e42b75e89f4dad6459200f92988c5998dfb093c7c631fb48524c86cd5dc", size = 10729260 }, + { url = "https://files.pythonhosted.org/packages/b0/d0/9c604bc72c37ba69b867b6df724d6a5af6789e8c375022c952f65b2af558/pyproj-3.7.1-cp313-cp313-win32.whl", hash = "sha256:d666c3a3faaf3b1d7fc4a544059c4eab9d06f84a604b070b7aa2f318e227798e", size = 5855462 }, + { url = "https://files.pythonhosted.org/packages/98/df/68a2b7f5fb6400c64aad82d72bcc4bc531775e62eedff993a77c780defd0/pyproj-3.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3caac7473be22b6d6e102dde6c46de73b96bc98334e577dfaee9886f102ea2e", size = 6266573 }, ] [[package]] @@ -2186,18 +2533,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/a6/145655273568ee78a581e734cf35beb9e33a370b29c5d3c8fee3744de29f/python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd", size = 8067 }, ] -[[package]] -name = "python-utils" -version = "3.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/99/fd1e3f80357dd88378281013ae7040a443de395bb0855bf17cbb828488d1/python_utils-3.9.0.tar.gz", hash = "sha256:3689556884e3ae53aec5a4c9f17b36e752a3e93a7ba2768c6553fc4dd6fa70ef", size = 35352 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/e7/200045de1f41ad74915a334f7c2aac54d2d6f4b89643a53e1062cde4895b/python_utils-3.9.0-py2.py3-none-any.whl", hash = "sha256:a7719a5ef4bae7360d2a15c13b08c4e3c3e39b9df19bd16f119ff8d0cfeaafb7", size = 32085 }, -] - [[package]] name = "pytz" version = "2024.2" @@ -2412,6 +2747,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, ] +[[package]] +name = "restructuredtext-lint" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/9c/6d8035cafa2d2d314f34e6cd9313a299de095b26e96f1c7312878f988eec/restructuredtext_lint-1.4.0.tar.gz", hash = "sha256:1b235c0c922341ab6c530390892eb9e92f90b9b75046063e047cacfb0f050c45", size = 16723 } + [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -2681,6 +3025,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a", size = 93002 }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + [[package]] name = "soupsieve" version = "2.6" @@ -2859,6 +3212,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521 }, ] +[[package]] +name = "stevedore" +version = "5.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pbr" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/3f/13cacea96900bbd31bb05c6b74135f85d15564fc583802be56976c940470/stevedore-5.4.1.tar.gz", hash = "sha256:3135b5ae50fe12816ef291baff420acb727fcd356106e3e9cbfa9e5985cd6f4b", size = 513858 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/45/8c4ebc0c460e6ec38e62ab245ad3c7fc10b210116cea7c16d61602aa9558/stevedore-5.4.1-py3-none-any.whl", hash = "sha256:d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe", size = 49533 }, +] + [[package]] name = "tabulate" version = "0.9.0" @@ -2868,6 +3233,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "tblib" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/95/4b3044ec4bf248186769629bbfb495a458deb6e4c1f9eff7f298ae1e336e/tblib-3.1.0.tar.gz", hash = "sha256:06404c2c9f07f66fee2d7d6ad43accc46f9c3361714d9b8426e7f47e595cd652", size = 30766 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/44/aa5c8b10b2cce7a053018e0d132bd58e27527a0243c4985383d5b6fd93e9/tblib-3.1.0-py3-none-any.whl", hash = "sha256:670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e", size = 12552 }, +] + [[package]] name = "terminado" version = "0.18.1" @@ -2882,6 +3256,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154 }, ] +[[package]] +name = "timezonefinder" +version = "6.5.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "h3" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/4d/8694391d6b014bc49f8fd2eb8c05c94526b36fba8dc76d439f3d51948e46/timezonefinder-6.5.9.tar.gz", hash = "sha256:0d84c792a499fd098a35c701c3e3293423ba8d45c81b3eecd7c7cb72c7f1f415", size = 51435008 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/d0/e24bdd01bfda4f66fac30e58a339d1d733718000b517c93ebfa792f946c0/timezonefinder-6.5.9-cp310-cp310-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52d5a4a8fc96990f72d9d3d48297e789217f67689d3178c9ff8ea3ab57125e3b", size = 51443226 }, + { url = "https://files.pythonhosted.org/packages/68/6a/321f849a41ac598981dfded6b9e2c0741b0406e6b5aa54f475058f156af4/timezonefinder-6.5.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a84ad5afb02ca1b536481cee05a8f2d5d7dd4818f73cd780acd03aa3cc033c9", size = 51445162 }, + { url = "https://files.pythonhosted.org/packages/0e/24/6b10ae9aae25871d6776c78b77322632b023f4d5a49d9c657f268a2ec568/timezonefinder-6.5.9-cp311-cp311-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:643535f76436b13216ed1c3b69c6ed8f793253810916ca6bef3b8e0cf3084fef", size = 51443229 }, + { url = "https://files.pythonhosted.org/packages/c8/15/89b0d616e4b6d92198a61019e7347a565a52b37ba3eed59178be00dcb8ab/timezonefinder-6.5.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4fa18e4b3f3ac1469bc50031700bc9696477cbf40953bdcbdec7bef20f3200d0", size = 51445199 }, + { url = "https://files.pythonhosted.org/packages/c9/aa/57fb0cb5b739a12ce6fc401b9d1e5d0b2ee092449c1d3ae14aa815409062/timezonefinder-6.5.9-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e9a0caf638f43b6dd9980731d1424500c7a6a5048db808aad7032560df5c663", size = 51443482 }, + { url = "https://files.pythonhosted.org/packages/2b/b9/c8a05e55096deea0b789d649da9729312ab1d97e3c52b0f0254c18f94cc7/timezonefinder-6.5.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c5347c4a73b40af4867a2946a5172ec68b644e7036888b9b5a0e568499bfc0f3", size = 51445521 }, + { url = "https://files.pythonhosted.org/packages/53/6c/498b3f453b15d7ecf2ecf54a8f1cc7fbd1d3c3eed86a5182081ebc17b180/timezonefinder-6.5.9-cp313-cp313-manylinux_2_17_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d1f30b550c24598459643285ef0f5469524ad7a32dd854199e7a2a463600ff", size = 51443403 }, + { url = "https://files.pythonhosted.org/packages/8b/16/6ae92a93dd703c0485e4c0a76e28bc829f9c7eba548a1b2e17fb3342e4c3/timezonefinder-6.5.9-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:4f36877ad3a988f329cbd3f04f7cadc56dce073ad4a7bda7397f46ac2a61f9a4", size = 51445432 }, +] + [[package]] name = "tinycss2" version = "1.3.0" @@ -2894,6 +3289,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/4d/0db5b8a613d2a59bbc29bc5bb44a2f8070eb9ceab11c50d477502a8a0092/tinycss2-1.3.0-py3-none-any.whl", hash = "sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7", size = 22532 }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] + [[package]] name = "tomli" version = "2.0.2" @@ -2935,7 +3339,7 @@ name = "tqdm" version = "4.66.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/83/6ba9844a41128c62e810fddddd72473201f3eacde02046066142a2d96cc5/tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad", size = 169504 } wheels = [ @@ -3034,16 +3438,25 @@ wheels = [ [[package]] name = "xarray" -version = "2024.9.0" +version = "2025.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "packaging" }, { name = "pandas" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/d3/ae7a92c8448c40cd43f97fff93b1a57f87565b412fdc02eb14af5d4c3823/xarray-2024.9.0.tar.gz", hash = "sha256:e796a6b3eaec11da24f33e4bb14af41897011660a0516fa4037d3ae4bbd1d378", size = 3747432 } +sdist = { url = "https://files.pythonhosted.org/packages/9b/29/37761364e137db13898cf5a790574dd7883f7355d5dfb42b66ee7a9a6318/xarray-2025.4.0.tar.gz", hash = "sha256:2a89cd6a1dfd589aa90ac45f4e483246f31fc641836db45dd2790bb78bd333dc", size = 2974151 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/1e/96fd96419fec1a37da998a1ca3d558f2cae2f6f3cd5015170371b05a2b6b/xarray-2025.4.0-py3-none-any.whl", hash = "sha256:b27defd082c5cb85d32c695708de6bb05c2838fb7caaf3f952982e602a35b9b8", size = 1290171 }, +] + +[[package]] +name = "zict" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/ac/3c494dd7ec5122cff8252c1a209b282c0867af029f805ae9befd73ae37eb/zict-3.0.0.tar.gz", hash = "sha256:e321e263b6a97aafc0790c3cfb3c04656b7066e6738c37fffcca95d803c9fba5", size = 33238 } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/28/3a6365e45721c7c9078968ed94b4a60076bc31d73b8519021a69b4995b63/xarray-2024.9.0-py3-none-any.whl", hash = "sha256:4fd534abdf12d5fa75dd566c56483d5081f77864462cf3d6ad53e13f9db48222", size = 1191607 }, + { url = "https://files.pythonhosted.org/packages/80/ab/11a76c1e2126084fde2639514f24e6111b789b0bfa4fc6264a8975c7e1f1/zict-3.0.0-py2.py3-none-any.whl", hash = "sha256:5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae", size = 43332 }, ] [[package]]