diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 4ec57294..6e973741 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -46,4 +46,14 @@ jobs: pip install pytest pip install setuptools pip install -e ".[download]" - pytest tests/pr/ + # Set environment variables for debugging + export PYTEST_CURRENT_TEST=1 + export DASK_DISTRIBUTED__DIAGNOSTICS__NVML=False + # Run pytest with verbose output and show all logs + pytest tests/pr/ \ + -v \ + --tb=long \ + --log-cli-level=DEBUG \ + --log-cli-format="%(asctime)s [%(levelname)8s] %(name)s: %(message)s" \ + --capture=no \ + -s diff --git a/.gitignore b/.gitignore index 303367aa..9e7cabab 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,6 @@ scripts/cluster # PDM .pdm-python + +# macos specific +.DS_Store diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f80269d6..12f27b85 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,13 +4,12 @@ build: os: "ubuntu-22.04" tools: python: "3.12" - -python: - install: - - method: pip - path: . - extra_requirements: - - docs + jobs: + # Avoid `pip install .[docs]`, which resolves geopandas, rasterio, dask, pvlib, + # etc. and often OOMs on RTD builders. Docs only need Sphinx + the source tree. + install: + - pip install -r docs/requirements.txt + - pip install --no-deps . sphinx: configuration: docs/source/conf.py diff --git a/README.md b/README.md index 2b1b912a..1b2bb668 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Installation will also install the following dependencies: * `bottleneck` * `numexpr` * `xarray` -* `netcdf4` +* `h5netcdf` * `dask` * `boto3` * `toolz` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..4a487e8a --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,8 @@ +# Sphinx stack (matches pyproject [project.optional-dependencies] docs) +sphinx>=8.0.0 +myst-nb>=1.1.2 +sphinx-book-theme>=1.1.3 +sphinx-autoapi==3.3.2 + +# Install geodata itself without pulling the full runtime dependency tree +# (see .readthedocs.yaml: pip install --no-deps .) diff --git a/docs/source/conf.py b/docs/source/conf.py index ea9d69e7..8262e956 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,12 +6,21 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -from geodata import __version__ +import re +from pathlib import Path + +# Read version without importing geodata (RTD installs with --no-deps). +_version_file = Path(__file__).resolve().parents[2] / "src" / "geodata" / "_version.py" +_release = re.search( + r'^__version__\s*=\s*["\']([^"\']+)["\']', _version_file.read_text(), re.M +) +if _release is None: + raise RuntimeError(f"Could not parse __version__ from {_version_file}") +release = _release.group(1) project = "Geodata" copyright = "2025, Geodata Contributors" author = "Geodata Contributors" -release = __version__ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/source/datasets/era5.rst b/docs/source/datasets/era5.rst index 2af0aee6..116cc9aa 100644 --- a/docs/source/datasets/era5.rst +++ b/docs/source/datasets/era5.rst @@ -1,81 +1,89 @@ ERA5 Specific Instructions ========================== -This page explains how you can set up access to ERA5 data from the `Copernicus Data Store `_. +This page covers **CDS account and API credential setup** for ERA5. Once credentials +are in place, use the dataset classes — do not call ``cdsapi`` by hand for routine +downloads. -Creating a CDS account ----------------------- - -To download ERA5 data from the CDS, you'll need to create a free `CDS account here `_. - -Download data through CDS API +Recommended download method ----------------------------- -Once your account has been created, set up access to the API by following these steps: +The **recommended way** to fetch ERA5 data in Geodata is: -1. Log into your CDS account and visit your `profile page `_. -2. Install the API key. There will be a section called **Personal Access Token**. - Copy these two lines into a file called ``.cdsapirc`` in your user root folder. +1. Complete the CDS setup below (one-time). +2. Follow :ref:`downloading-era5-data` in :doc:`overview` — ``load_dataset``, + instantiate with ``years`` / ``months`` / optional ``bounds``, then ``download()``. -- **macOS/Linux**: Open a terminal and run: +Geodata's ERA5 classes (for example ``ERA5Wind3DHourlyDataset``) create a +``cdsapi.Client`` internally and submit the correct product requests for each +registered ``weather_config``. - .. code-block:: bash +Creating a CDS account +---------------------- - touch ~/.cdsapirc +To download ERA5 data from the CDS, create a free `CDS account here `_. - Then add the lines using: +Configure CDS API credentials +----------------------------- - .. code-block:: bash +Once your account exists, install local API access: - echo [line 1 of the code] >> ~/.cdsapirc - echo [line 2 of the code] >> ~/.cdsapirc +1. Log into your CDS account and visit your `profile page `_. +2. Under **Personal Access Token**, copy the two lines for your ``.cdsapirc`` file + (URL and key). +**macOS/Linux** — create ``~/.cdsapirc``: - - **Windows**: The process is slightly more complicated. Please refer to the in-depth guide at the Copernicus Knowledge Base `here `_. +.. code-block:: bash -3. Install the CDS API client by opening a terminal/shell and running + touch ~/.cdsapirc + # Paste the two lines from your CDS profile into ~/.cdsapirc -.. code-block:: bash +**Windows** — see the Copernicus guide on +`installing the CDS API on Windows `_. - pip install ".[download]" +Ensure ``cdsapi`` is available (it is a dependency of Geodata when you install the +package). Then proceed to :ref:`downloading-era5-data` in :doc:`overview`. -(Assuming you are in Geodata's *root directory*.) +Verify CDS API access (optional) +-------------------------------- -1. Once you've installed the API key and the API client, confirm access by running an - example in a Python script or a Jupyter notebook: +You can confirm credentials with a minimal ``cdsapi`` script. This is **optional** — +Geodata dataset downloads use the same client and credentials. .. code-block:: python - import cdsapi - - c = cdsapi.Client() - - c.retrieve( - "reanalysis-era5-single-levels", - { - "product_type": "reanalysis", - "format": "netcdf", - "variable": [ - "2m_dewpoint_temperature", - "2m_temperature", - ], - "year": "2011", - "month": [ - "01", - ], - "day": ["01", "02", "03"], - "time": [ - "00:00", - "12:00", - ], - }, - "download.nc", - ) - -The above example downloads 2m temperature and 2m dewpoint temperature with data points -at 00:00 and 12:00 for each day, from January 1-3, 2011, in NetCDF format. - -If this works, you have successfully set up access to the ERA5 data through the CDS API. -Please subsequently refer to the `general documentation on datasets <../overview.rst>`_ -for more information on how to download ERA5-based datasets using the ``geodata`` -package. + import cdsapi + + c = cdsapi.Client() + + c.retrieve( + "reanalysis-era5-single-levels", + { + "product_type": "reanalysis", + "format": "netcdf", + "variable": [ + "2m_dewpoint_temperature", + "2m_temperature", + ], + "year": "2011", + "month": ["01"], + "day": ["01", "02", "03"], + "time": ["00:00", "12:00"], + }, + "download.nc", + ) + +This example fetches 2 m temperature and dewpoint at 00:00 and 12:00 UTC for +2011-01-01 through 2011-01-03. If it succeeds, your ``.cdsapirc`` is valid. + +For production workflows, prefer :ref:`downloading-era5-data` in :doc:`overview` so +Geodata requests the correct ERA5 products, paths, and post-processing for +``wind_3d_hourly``, ``wind_solar_hourly``, and other registered configs. + +What's next +----------- + +- :ref:`downloading-era5-data` in :doc:`overview` — **recommended** download workflow +- :doc:`../development/offline-era5-fixture-datasets` — offline ``*_test`` configs for CI +- :doc:`../modeling/wind/index` or :doc:`../modeling/pvlib/index` — run models on downloaded data diff --git a/docs/source/datasets/overview.rst b/docs/source/datasets/overview.rst index 57430059..d6622e48 100644 --- a/docs/source/datasets/overview.rst +++ b/docs/source/datasets/overview.rst @@ -9,85 +9,127 @@ data formats, handling metadata, and performing common geospatial operations. Key Features ------------ -- Supports the download and management of datasets from various sources, such as - `ERA5 `_ and - `MERRA2 `_. +- Supports the download and management of **ERA5** datasets via ``load_dataset`` (see :doc:`era5` for CDS account setup). -- Provides a consistent API for accessing geospatial data, regardless of the underlying - data source. +- **MERRA2** remains in the codebase but is documented under :doc:`/legacy/index` (legacy ``Dataset`` / ``Cutout`` path, not part of the current tested workflow). Typical Usage ------------- -In the following example, we will demonstrate how to download a dataset containing wind -and solar data from ECMWF's ERA5 dataset. +Registered datasets are loaded by name, instantiated with a time range (and optional +geographic bounds), then downloaded with ``download()`` if the files are not already +on disk. The sections below use **ERA5** as the primary example; the same pattern +applies to other configs returned by ``list_datasets()``. + +.. _downloading-era5-data: + +Downloading ERA5 data +--------------------- + +Geodata's ERA5 dataset classes wrap the `Copernicus CDS API `_. +You configure credentials once (see :doc:`era5`), then download through Python — you do +**not** need to call ``cdsapi`` directly for normal use. + +Files are stored under ``GEODATA_ROOT / era5 / / …`` (see +:doc:`../quick_start/packagesetup` for ``GEODATA_ROOT``). + +**Example — 3D wind (for :doc:`../modeling/wind/index`):** .. code-block:: python from geodata.datasets import load_dataset - dataset_cls = load_dataset("wind_solar_hourly") + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( + years=slice(2016, 2016), + months=slice(1, 1), + bounds=[-10, 35, 10, 45], # lon_min, lat_min, lon_max, lat_max + ) + + print(ds.downloaded) # False until files exist locally + + if not ds.downloaded: + ds.download() + + print(ds.downloaded) # True when the catalog is complete + +**Example — 2D wind and solar hourly (for :doc:`../modeling/pvlib/index`):** + +.. code-block:: python + + from geodata.datasets import load_dataset - years = slice(2010, 2020) - months = slice(1, 13) - dataset = dataset_cls(years=years, months=months) + ds_cls = load_dataset("wind_solar_hourly") + ds = ds_cls(years=slice(2016, 2016), months=slice(1, 1)) -Here, we first create a dataset class using the `load_dataset` function, specifying the -name of the dataset we want to load. We then instantiate the dataset class with the -desired time range (years and months). Then, we can create a dataset instance with -that class, which will handle the downloading and processing of the data. + if not ds.downloaded: + ds.download() + +``bounds`` is optional; omit it to use the full spatial extent allowed by the dataset +class. With ``testing=True``, only a **small subset** of the catalog is requested (useful +for trying a download before committing to a full month): + +.. code-block:: python + + ds = ds_cls( + years=slice(2016, 2016), + months=slice(1, 1), + bounds=[50, 0, 48, 3], + testing=True, + ) + ds.download() + +After ``downloaded`` is ``True``, pass ``ds`` to a model (for example +``WindInterpolationModel(ds)`` or ``Pvlib(ds)``). + +Offline / CI without CDS +~~~~~~~~~~~~~~~~~~~~~~~~ + +For tests and local development without calling the CDS, use the committed fixture +configs ``wind_3d_hourly_test`` and ``wind_solar_hourly_test`` — same API, no +``download()`` required when fixture files are present. See +:doc:`../development/offline-era5-fixture-datasets`. Dataset Classes ----------------- -The `geodata.datasets` module includes several dataset classes, each tailored for + +The ``geodata.datasets`` module includes several dataset classes, each tailored for specific datasets. These classes encapsulate the logic for downloading, processing, and -accessing the data. Some of the available dataset classes -(listed by `weather_data_config`) include: +accessing the data. Some of the available ERA5 configs +(listed by ``weather_config``) include: -- `wind_solar_hourly`: A dataset containing hourly wind and solar data from ECMWF's - ERA5. It is important to note that the wind data are only recorded at - 10 and 100 meters above ground level. Hence, this dataset is also referred to as - 2D wind and solar dataset. +- ``wind_solar_hourly``: Hourly wind (10 m and 100 m) and solar radiation from ERA5 + single levels. Also referred to as the 2D wind and solar dataset. -- `wind_3d_hourly`: A dataset containing hourly wind data from ECMWF's ERA5 at - multiple vertical levels, providing a more comprehensive view of the wind profile. - It can be used for wind speed estimation using and interpolation model built into - the geodata library. +- ``wind_3d_hourly``: Hourly wind on ERA5 model levels (131–137), stored as **daily** + NetCDF files. Used by the wind interpolation model for hub-height wind speed. -You can use the `list_datasets` function to see all available datasets in the -`geodata.datasets` module. This function returns a list of dataset names that can be -loaded using the `load_dataset` function. For example: +You can use ``list_datasets()`` to see all registered names: .. code-block:: python from geodata.datasets import list_datasets - available_datasets = list_datasets() - print(available_datasets) # Outputs a list of available dataset names. - -Check Preparedness of Datasets ------------------------------------------------- -To check if a dataset is prepared and ready for use, you can use the `downloaded` -property of the dataset instance. This property returns a boolean indicating whether the -dataset is fully prepared. If the dataset is not prepared, you can call the `prepare` -method to download and process the data. For example: + print(list_datasets()) -.. code-block:: python +Check whether data is on disk +------------------------------ - print(dataset.downloaded) # Check if the dataset is downloaded. Outputs False here. +The ``downloaded`` property is ``True`` when every file in the dataset **catalog** exists +under ``storage_root``. If any file is missing, call ``download()`` (or ``download(force=True)`` +to re-fetch): - if not dataset.downloaded: - dataset.download() +.. code-block:: python - print(dataset.downloaded) # Outputs True after downloading. + if not ds.downloaded: + ds.download() Dataset's Interoperability with Cutout ------------------------------------------------ -At the moment, the dataset classes are not interoperable with the `Cutout` class. -In the future, we plan to consolidate the functionalities of the `Cutout` class into the -dataset classes and the modeling module (see :doc:`here<../modeling/wind/index>`). +At the moment, the dataset classes are not interoperable with the ``Cutout`` class. +In the future, we plan to consolidate the functionalities of the ``Cutout`` class into the +dataset classes and the modeling module (see :doc:`../modeling/wind/index`). -For now, after downloading a dataset, a good point to move forward would be to use the -:doc:`modeling module <../modeling/index>` to create a model that can do certain types -of modeling with the dataset, such as wind speed estimation or solar PV generation. +For now, after downloading a dataset, pass it to a modeling class — see +:doc:`../modeling/wind/index` or :doc:`../modeling/pvlib/index`. diff --git a/docs/source/datasets/weather_data_config.md b/docs/source/datasets/weather_data_config.md index 098c64a4..8dcf9748 100644 --- a/docs/source/datasets/weather_data_config.md +++ b/docs/source/datasets/weather_data_config.md @@ -4,8 +4,10 @@ In Geodata, every downloadable dataset are associated with a unique `(module, we In this tuple, the `module` typicallly refers to the source of dataset, while the `weather_data_config` is a dictionary that contains the information needed to download the specific form of the dataset. -As Geodata currently supports `ERA5` and `MERRA2` modules, you can find all relevant weather data configuration -in each module's introduction pages here ([ERA5](era5/index.md), [MERRA2](merra2/index.md)). To find each config's actual definition, you can go to `src/geodata/datasets`. Within it, all available weather data configurations are located at the bottom of the file. +Geodata supports ERA5 through the modern `load_dataset` registry and MERRA2 through the legacy +`Dataset(module="merra2", ...)` API. Introduction pages: [ERA5](era5.rst), +[MERRA2 (legacy)](../legacy/merra2/index.md). To find each config's actual definition, go to +`src/geodata/datasets` — weather data configurations are defined at the bottom of each module file. In this tutorial, we will discuss the structure of each `weather_data_config` in more details. diff --git a/docs/source/development/documentation-organization-plan.md b/docs/source/development/documentation-organization-plan.md new file mode 100644 index 00000000..8d8ca91c --- /dev/null +++ b/docs/source/development/documentation-organization-plan.md @@ -0,0 +1,396 @@ +# Geodata documentation organization plan + +This document defines how Geodata documentation is structured, how it maps to +`src/geodata`, and how we keep prose, notebooks, and API reference aligned as +the library evolves. It is intended for contributors working on the +**documentation branch** and for anyone opening a PR that changes user-facing +behavior. + +**Status:** living plan (update this file when conventions change). + +--- + +## 1. Goals + +1. **One clear user journey** — readers should know whether to use the modern + (`load_dataset` → model → optional masking) or legacy (`Dataset` → `Cutout` → + `convert`) workflow without reading the entire site. +2. **Docs follow code** — every public API change in `src/` has a defined doc + touchpoint (prose, notebook, or autoapi docstring). +3. **No orphan pages** — every `.md`, `.rst`, and `.ipynb` under + `docs/source/` appears in a `toctree` or is explicitly marked as internal + (see [Section 5](#5-file-types-and-conventions)). +4. **Reproducible examples** — tutorials should run offline where possible + (ERA5 `*_test` fixtures) so CI and local builds do not depend on CDS + credentials. +5. **Separation of concerns** — migration plans and design notes stay in + `development/` or clearly labeled plan pages; user-facing tutorials stay + task-focused. + +--- + +## 2. Current state (baseline) + +### 2.1 Two parallel workflows + +Geodata currently exposes two stacks. Both are valid; documentation must label +them explicitly. + +| Aspect | Modern workflow | Legacy workflow | +|--------|-----------------|-----------------| +| Data access | `geodata.datasets.load_dataset(...)` | `geodata.Dataset(module=..., weather_data_config=...)` | +| Subsetting | `BaseDataset` bounds / model `xs`/`ys` | `geodata.Cutout` + `prepare()` | +| Transform | `geodata.model.wind`, `geodata.model.pvlib` | `geodata.convert.*` on Cutouts | +| Masking (apply) | `geodata.XarrayMask` | `cutout.add_mask()` + `cutout.mask()` | +| Masking (create) | `geodata.Mask` (same for both) | `geodata.Mask` (same for both) | +| Primary docs | `datasets/`, `modeling/` | `intro.rst`, mask Cutout notebooks | + +**Canonical path for new features:** modern workflow. Legacy paths remain +documented until explicitly deprecated. + +### 2.2 Documentation build stack + +| Piece | Location | Role | +|-------|----------|------| +| Sphinx config | `docs/source/conf.py` | MyST, notebooks, autoapi | +| Site root | `docs/source/index.rst` | Top-level toctrees | +| Landing narrative | `docs/source/intro.rst` | Overview (still legacy-heavy) | +| API reference | autoapi → `src/geodata` | Generated from docstrings | +| Notebooks | `myst_nb`, `nb_execution_mode = "off"` | Committed outputs; not executed on build | + +### 2.3 Known gaps (as of this plan) + +| Gap | Impact | Priority | Status | +|-----|--------|----------|--------| +| `intro.rst` teaches legacy Cutout/convert as the main story | New users miss models + `XarrayMask` | P0 | **Done** — modern intro on homepage; legacy moved to `legacy/workflow.rst` | +| Modeling pages missing recent API options (`compact_output`, flexible `xs`/`ys`) | Docs diverge from `src` | P0 | **Done** — see modeling/wind/index and modeling/pvlib/index | +| Wind capacity-factor internals not in wind toctree | Deep-dive exists only in source/comments | P1 | **Done (Option A)** — “Understanding the output” in interpolation/extrapolation Step 5 | +| `xarray_mask_tutorial.ipynb` referenced by `xarray_mask_workflow.rst` but may be missing from tree | Broken `:doc:` link | P0 | **Done** — notebook added under `mask/` | +| Mask section mixes user tutorials with `mask_xarray_migration_plan.md` | Hard to tell “how-to” vs “plan” | P1 | **Done** — plans moved to `development/` | +| `development/offline-era5-fixture-datasets.md` not linked from modeling tutorials | Readers assume CDS required | P1 | **Done** — overview + era5 + intro link fixtures | +| Example scripts in `docs/source/mask/*.py` not classified | Unclear if maintained or one-off | P2 | Open | +| README points to placeholder doc URL | External discoverability | P2 | Open | + +--- + +## 3. Target information architecture + +Organize the site by **user task**, not by file type. Recommended sidebar +structure (matches `index.rst` with clearer intent): + +``` +Geodata docs +├── Getting started +│ ├── Package setup +│ ├── Supported I/O formats +│ └── Workflow chooser (NEW — short page: modern vs legacy) +├── Datasets +│ ├── Overview (load_dataset, list_datasets) +│ ├── ERA5 (CDS setup + configs) +│ ├── MERRA2 +│ └── Weather data config reference +├── Modeling +│ ├── Wind (index + interpolation + extrapolation; CF notes in Step 5) +│ └── PVLib (index + future subpages) +├── Masking +│ ├── Create masks (mask_creation_workflow.ipynb) +│ ├── Apply with XarrayMask (xarray_mask_tutorial.ipynb) +│ ├── Troubleshoot (mask_troubleshoot.md) +│ └── Cutout apply → legacy/mask_on_cutout +├── Visualization +├── Development (contributors) +│ ├── Documentation organization (this file) +│ ├── Offline ERA5 fixtures +│ ├── mask_xarray_migration_plan +│ └── xarray_mask_workflow (implementation notes) +└── API reference (autoapi) +``` + +### 3.1 Page roles (Diátaxis) + +Use four doc types consistently: + +| Type | Purpose | Examples | +|------|---------|----------| +| **Tutorial** | Learning-oriented, step-by-step | Notebooks, `modeling/wind/interpolation.rst` | +| **How-to guide** | Goal-oriented recipe | `xarray_mask_tutorial.ipynb`, ERA5 CDS setup | +| **Reference** | Accurate, complete | autoapi, `weather_data_config.md`, turbine YAML lists | +| **Explanation** | Concepts and design | migration plans, wind CF summary in interpolation Step 5 | + +Label migration/plan documents at the top: + +```markdown +> **Audience:** contributors and maintainers. For usage, see [XarrayMask tutorial](../mask/xarray_mask_tutorial.ipynb). +``` + +--- + +## 4. Source code ↔ documentation map + +Maintain this table when adding modules. **Primary doc** is the page that must +be updated first when behavior changes. + +| `src/geodata` area | Primary doc | Secondary / API | +|--------------------|-------------|-----------------| +| `datasets/_base.py`, `datasets/era5/*` | `datasets/overview.rst`, `datasets/era5.rst` | autoapi | +| `datasets/merra2/*` (legacy) | `legacy/merra2/*` | autoapi | +| `datasets/era5/fixture.py` (`*_test`) | `development/offline-era5-fixture-datasets.md` | modeling tutorials (offline note) | +| `model/wind/*` | `modeling/wind/index.rst`, `interpolation.rst`, `extrapolation.rst` | autoapi | +| `model/pvlib/_base.py` | `modeling/pvlib/index.rst` | autoapi | +| `model/_base.py` (slice sel, I/O) | modeling pages (bounding box sections) | autoapi | +| `mask.py` (legacy Mask) | `mask/mask_creation_workflow.ipynb` | autoapi | +| `mask/xarray_mask.py`, `mask/spatial.py` | `mask/xarray_mask_tutorial.ipynb` | `development/xarray_mask_workflow.rst` (contributors) | +| `cutout.py`, `convert.py`, `preparation.py` | `legacy/workflow.rst` | autoapi | +| Cutout-based masking | `legacy/mask_on_cutout.ipynb` | — | +| `plot.py` | `visualization/visualization.ipynb` | autoapi | +| `resource.py`, `resources/*` | modeling pages (turbine/panel names) | — | +| `config.py` | `quick_start/packagesetup.md` | — | + +### 4.1 Public exports (`__init__.py`) + +When adding or removing symbols from `geodata.__all__`: + +1. Update docstrings (autoapi). +2. Update `intro.rst` or the relevant tutorial if the symbol is part of a + documented workflow. +3. Add a line to the [changelog section](#72-changelog-expectations) of the PR. + +--- + +## 5. File types and conventions + +### 5.1 Where files live + +| Path | Use for | +|------|---------| +| `docs/source/quick_start/` | Install, env vars, I/O formats | +| `docs/source/datasets/` | Download, configs, dataset-specific outputs | +| `docs/source/modeling//` | Model tutorials and domain index | +| `docs/source/mask/` | Mask tutorials, workflows, troubleshooting | +| `docs/source/visualization/` | Plotting notebooks | +| `docs/source/development/` | Contributor docs, fixtures, **this plan**, internal design | +| `docs/source/_static/` | Images referenced from rst/md | + +### 5.2 Format choice + +| Format | When to use | +|--------|-------------| +| `.rst` | Sphinx-native pages with toctrees (section indexes) | +| `.md` (MyST) | Prose guides, plans, troubleshooting | +| `.ipynb` | Executable narratives with plots; keep outputs committed | + +### 5.3 Naming + +- User-facing: `snake_case` or `kebab-case` descriptive names + (`xarray_mask_tutorial.ipynb`, `mask_troubleshoot.md`). +- Plans: suffix or folder under `development/` (`*_plan.md`, `*_known_issues.md`). +- Example scripts: `docs/source//examples/` (proposed) — not mixed with + built pages unless listed in toctree. + +### 5.4 Internal vs published pages + +Pages under `development/` and `mask/*_plan.md` are **contributor-facing**. +They stay in the toctree under **Development** or with an audience banner so +users are not sent to migration checklists by mistake. + +Optional future convention: prefix internal-only files with `_` and exclude in +`conf.py` `exclude_patterns` — not required if audience banners are used. + +--- + +## 6. Keeping documentation up to date + +### 6.1 PR checklist (code changes) + +Every PR that changes `src/geodata` should answer: + +- [ ] Does this change **public API** or default behavior? +- [ ] Which **primary doc** row in [Section 4](#4-source-code--documentation-map) applies? +- [ ] Are **docstrings** updated for autoapi? +- [ ] Is there a **minimal code snippet** in prose docs or a test that can be copied? +- [ ] Do **notebooks** need re-run outputs (if affected)? +- [ ] Does `intro.rst` need a **workflow label** (modern vs legacy) if touched? + +If the answer to the first question is yes and no doc file is updated, the PR +should either include doc updates or link a follow-up issue. + +### 6.2 Documentation-only PRs (this branch) + +Recommended batching for the documentation branch: + +| Phase | Work | Outcome | +|-------|------|---------| +| **A — Structure** | Workflow chooser; relabel legacy in `intro.rst`; wire orphan pages into toctrees | Clear navigation | +| **B — Sync with recent `src`** | `XarrayMask`, `compact_output`, slice/bounds notes, fixture offline path | Factual parity with code | +| **C — Depth** | Wind CF explanation, mask troubleshooting, merge-layer known issues | Explanation layer | +| **D — Hygiene** | Move example `.py` to `examples/`; README doc URL; trim stale “planned” notes | Lower maintenance cost | + +### 6.3 When to update which layer + +| Change in `src` | Update prose/notebook | Update docstrings only | +|-----------------|----------------------|-------------------------| +| New public class or method | Yes | Yes | +| New optional parameter with non-obvious default | Yes (one example) | Yes | +| Internal refactor, same API | No | Only if signatures changed | +| Bug fix affecting coordinates, units, or outputs | Yes (note in troubleshooting or tutorial) | Yes | +| New `*_test` fixture config | `development/offline-era5-fixture-datasets.md` | — | +| Deprecation | Yes + migration plan | Yes | + +### 6.4 Single source of truth + +| Content | Source of truth | Docs should… | +|---------|-----------------|--------------| +| Function signatures | `src/` + autoapi | Not duplicate parameter lists | +| End-to-end workflows | Notebooks + rst tutorials | Link to tests under `tests/pr/` | +| Dataset registry names | `list_datasets()` / `datasets/registry` | Regenerate or manually sync lists in `overview.rst` when configs added | +| Turbine/panel names | `resources/windturbine/`, `resources/solarpanel/` | Show representative examples, not full catalogs | + +Prefer **short examples from tests** over hand-written snippets that drift: + +```python +# Pattern: tests/pr/test_xarray_mask.py → docs/source/mask/xarray_mask_tutorial.ipynb +``` + +### 6.5 Build and review + +Local build: + +```bash +cd docs && make html +# open _build/html/index.html +``` + +Before merging the documentation branch: + +1. `make html` completes without warnings for missing `:doc:` references. +2. New pages appear in the correct toctree (sidebar). +3. Notebooks render (committed outputs present; `nb_execution_mode` is `off`). +4. autoapi pages generate for new modules. + +Future CI enhancements (optional): + +- Sphinx `-W` (warnings as errors) on PRs touching `docs/`. +- Link check for internal `:doc:` and relative md links. +- Script to diff `list_datasets()` output against `overview.rst` mentions. + +--- + +## 7. Immediate backlog for the documentation branch + +Actionable items in recommended order. + +### P0 — Navigation and broken links + +1. ~~**Add workflow chooser**~~ — **Done:** homepage (`intro.rst`) is the modern workflow; legacy content lives under **Legacy workflow** (`legacy/workflow.rst`). +2. ~~**Ensure `xarray_mask_tutorial.ipynb` exists**~~ — **Done** (`docs/source/mask/xarray_mask_tutorial.ipynb`, included via mask `*` toctree). +3. ~~**Update `intro.rst` masking section**~~ — **Done:** modern intro uses `XarrayMask`; Cutout masking unchanged in `legacy/workflow.rst`. + +### P0 — Sync with recent source changes + +4. ~~**`modeling/pvlib/index.rst`** — document `compact_output`~~ — **Done**. +5. ~~**`modeling/wind/index.rst` and interpolation.rst** — document flexible `xs`/`ys`~~ — **Done**. +6. ~~**`datasets/era5.rst`** — clarify CDS download vs offline fixtures~~ — **Done:** overview has recommended download; era5.rst covers CDS setup + optional cdsapi verify. + +### P1 — Structure and depth + +7. ~~**Wind CF documentation**~~ — **Done (Option A):** expanded Step 5 in interpolation/extrapolation; no separate internals page. +8. ~~**Reorganize mask toctree intent**~~ — **Done:** Mask = creation + apply tutorials + troubleshoot; plans under Development. +9. **`mask/merge_layer_known_issues.md`** — publish under mask with troubleshooting cross-links. +10. **Link fixture doc from modeling tutorials** — one paragraph + code using `load_dataset("wind_3d_hourly_test")`. + +### P2 — Hygiene + +11. Move `docs/source/mask/create_mask.py`, `split_china.py`, etc. to `docs/source/mask/examples/` (exclude from glob toctree or document as examples). +12. Fix README documentation URL placeholder. +13. Audit `input_output.md` “planned” notes against `list_datasets()`. +14. Add **this plan** to `index.rst` Development toctree (done when this file is merged). + +--- + +## 7.2 Changelog expectations + +Documentation PRs should summarize: + +- **User-visible:** what readers can now do or what corrected behavior is documented. +- **Structural:** new pages, moved pages, deprecated paths. +- **Not required:** typo fixes only. + +For paired code+doc releases, use a single changelog entry covering both. + +--- + +## 8. Long-term governance + +### 8.1 Ownership (suggested) + +| Area | Default maintainer focus | +|------|--------------------------| +| Datasets / ERA5 fixtures | Whoever changes `datasets/era5/` | +| Wind / PV modeling | Model module authors | +| Mask / XarrayMask | Mask package authors | +| Legacy Cutout/convert | Touch only when behavior changes; avoid new features here | + +### 8.2 Deprecation policy for docs + +When deprecating APIs: + +1. Mark in docstring + autoapi. +2. Add “Deprecated” admonition in legacy tutorial. +3. Record timeline in a `development/` plan or release notes. +4. Remove legacy tutorial sections only after code removal or major version bump. + +### 8.3 Quarterly doc audit (lightweight) + +Every ~3 months or before a release: + +1. Run `list_datasets()` and compare to `datasets/overview.rst`. +2. Scan `intro.rst` for legacy-only examples without modern pointers. +3. Grep docs for `planned`, `TODO`, `FIXME`. +4. Confirm `make html` clean build. +5. Update [Section 2.3](#23-known-gaps-as-of-this-plan) gap table in this file. + +### 8.4 Relationship to API reference + +autoapi is the **reference layer**; tutorials should not duplicate every +argument. Convention: + +- Tutorials: one worked example with common options. +- Reference: full signatures via docstrings (NumPy style, `sphinx.ext.napoleon`). +- Explanation pages: algorithms and data flow (e.g. wind CF pipeline). + +When adding a feature, **docstring first**, then **one tutorial paragraph** — +not a third full copy in markdown. + +--- + +## 9. Appendix: proposed `index.rst` Development toctree + +```rst +.. toctree:: + :maxdepth: 1 + :caption: Development + :hidden: + + development/documentation-organization-plan + development/offline-era5-fixture-datasets +``` + +Mask migration plan remains under `mask/` glob but should use a contributor +banner (see [Section 3.1](#31-page-roles-diátaxis)). + +--- + +## 10. Appendix: doc branch merge strategy + +1. **Land structure first** (toctrees, workflow chooser, intro labels) so follow-up edits have a home. +2. **Land content sync** (modeling/mask/datasets factual updates) in the same branch or stacked PRs by area. +3. **Avoid** mixing large narrative rewrites with unrelated code changes — keeps review focused. +4. After merge, tag a docs release note listing: new XarrayMask path, fixture-based tutorials, deprecated/legacy labeling. + +--- + +## Document history + +| Date | Change | +|------|--------| +| 2026-06-02 | Initial organization plan for documentation branch | diff --git a/docs/source/development/mask_xarray_migration_plan.md b/docs/source/development/mask_xarray_migration_plan.md new file mode 100644 index 00000000..7b3afb70 --- /dev/null +++ b/docs/source/development/mask_xarray_migration_plan.md @@ -0,0 +1,185 @@ +# Mask-Without-Cutout Migration Plan + +```{note} +**Audience:** contributors and maintainers. For applying saved masks to model +output, see [XarrayMask tutorial](../mask/xarray_mask_tutorial.ipynb). +``` + +## Goal + +Replace Cutout-dependent masking with a direct xarray-based workflow: + +- `datasets -> models -> masking -> analysis` + +The new masking flow should work on model output (`xarray.Dataset` / `xarray.DataArray`) directly, while reusing current `mask.py` code as much as possible. + +## What Changes, What Stays + +- Keep: + - `Mask` object for raster/shapefile mask creation and persistence. + - Existing layer operations in `src/geodata/mask.py` (`add_layer`, `filter_layer`, `merge_layer`, `extract_shapes`, `save_mask`, `from_name`). + - Existing geospatial utilities (`ras_to_xarr`, `calc_grid_area` logic from `cutout.py`, coordinate formatting helpers). +- Remove dependency on: + - `Cutout.add_mask(...)` + - `Cutout.add_grid_area(...)` + - `Cutout.mask(...)` +- Add: + - A new xarray-focused masking adapter class/module (proposed below). + +## Proposed Target API + +Create a dedicated class (example name: `XarrayMask`) that only deals with xarray data: + +1. **Creation / loading** + - `XarrayMask.from_mask(mask: Mask, grid: xr.Dataset | xr.DataArray, include_merged=True, include_shapes=True)` + - `XarrayMask.from_name(name: str, grid: xr.Dataset | xr.DataArray, mask_dir=...)` +2. **Area calculation** + - `XarrayMask.compute_grid_area(grid: xr.Dataset | xr.DataArray) -> xr.DataArray` +3. **Applying mask** + - `XarrayMask.attach(dataset, include_area=True) -> dict[str, xr.Dataset]` + - Equivalent to current `Cutout.mask(...)` behavior (mask as extra variables). + - `XarrayMask.apply(dataset, mode="where", include_area=False) -> dict[str, xr.Dataset]` + - New convenience method returning mask-applied outputs: + - `mode="where"`: outside mask -> NaN + - `mode="multiply"`: outside mask -> 0 + +This gives both: +- transparent feature-style behavior (`attach`) +- direct filtered outputs (`apply`) + +## Reuse Map (Do Not Reinvent) + +Directly reuse existing code paths: + +- From `src/geodata/mask.py`: + - `Mask.from_name(...)` + - `Mask.load_merged_xr()` / `Mask.load_shape_xr()` +- From `src/geodata/cutout.py`: + - `ds_reformat_index(...)` (move/shared helper) + - `coarsen(...)` (move/shared helper) + - `calc_grid_area(...)` (move/shared helper) +- Keep the same coordinate conventions: + - normalize to `lat`, `lon` + - align mask grid to target dataset grid before applying + +Refactor suggestion: +- Move shared helpers into a new utility module, e.g. `src/geodata/spatial.py` or `src/geodata/mask_xarray.py`, then import from both old and new flows during transition. + +## Migration Phases + +### Phase 0 - Freeze Current Behavior + +- Add tests that lock existing behavior for: + - coarsening/alignment from mask raster to target grid + - area computation + - output structure currently returned by `Cutout.mask(...)` + +This prevents regressions while extracting logic. + +### Phase 1 - Extract Shared Spatial Helpers + +- Move (or duplicate temporarily) these functions out of `cutout.py`: + - `ds_reformat_index` + - `coarsen` + - `calc_grid_area` +- Add unit tests for each helper independent of `Cutout`. + +### Phase 2 - Introduce `XarrayMask` + +- Implement class that: + - loads saved `Mask` by name + - converts mask rasters to xarray + - coarsens/aligned to target grid + - computes area from target grid + - provides `attach()` and `apply()` + +### Phase 3 - Integrate into datasets -> models workflow + +- At model output point (where xarray result exists), call: + - `xmask = XarrayMask.from_name("my_mask", grid=model_ds)` + - `masked = xmask.apply(model_ds, mode="where")` +- Keep `attach()` available for advanced users needing raw mask + area features. + +### Phase 4 - Deprecate Cutout Masking Surface + +- Mark these as deprecated: + - `Cutout.add_mask` + - `Cutout.add_grid_area` + - `Cutout.mask` +- Keep them as wrappers calling new `XarrayMask` for 1-2 releases. + +### Phase 5 - Remove Cutout Dependency + +- Remove or archive old mask-coupled Cutout paths once internal usage is migrated. +- Keep `Cutout` only if still needed for data preparation. + +## Detailed Behavior Decisions + +To avoid ambiguity, define these explicitly: + +- Mask value semantics: + - `mask > 0` means valid/included + - `mask <= 0` means excluded +- Apply scope: + - apply to all data variables by default + - optional include/exclude variable list +- Output keys: + - `"merged_mask"` for merged mask + - shape names for shape masks (same as current behavior) +- Alignment: + - always reformat coords to `lat`/`lon` + - always transpose to `time, lat, lon` when `time` exists +- Area: + - computed from target grid only (not from mask grid) to stay consistent with model outputs + +## Risks and Mitigations + +- Risk: hidden coordinate mismatches (`x/y` vs `lat/lon`, descending latitude). + - Mitigation: centralize coordinate normalization in one helper and test with both styles. +- Risk: users depending on old `Cutout.mask` output shape. + - Mitigation: make `attach()` output identical structure and keep temporary wrappers. +- Risk: performance hit when repeatedly coarsening same mask. + - Mitigation: cache aligned masks keyed by grid signature (lat/lon hashes + mask name). + +## Suggested Minimal First Milestone (1 PR) + +- Add `src/geodata/mask_xarray.py` with: + - `XarrayMask.from_name(...)` + - `compute_grid_area(...)` + - `attach(...)` + - `apply(...)` (`where` + `multiply`) +- Reuse copied helper logic from `cutout.py` initially (refactor later). +- Add tests: + - parity test with `Cutout.mask(...)` behavior for `attach()` + - correctness test for `apply(...)` + - area calculation sanity test + +## Example Future Usage + +```python +import geodata + +# model output +ds_model = model.run(...) # xr.Dataset with dims time/lat/lon (or x/y) + +# load and align mask to ds_model grid +xmask = geodata.XarrayMask.from_name("china", grid=ds_model) + +# 1) feature-style output (raw + mask + area) +attached = xmask.attach(ds_model, include_area=True) + +# 2) direct masked output +masked = xmask.apply(ds_model, mode="where", include_area=True) +china_masked = masked["merged_mask"] +``` + +## Recommended Naming + +- Keep existing `Mask` name for geospatial mask construction object. +- Use a distinct name for xarray adapter to avoid confusion: + - preferred: `XarrayMask` + - alternatives: `MaskApplier`, `MaskDatasetAdapter` + +This separation keeps responsibilities clear: +- `Mask`: build/store masks +- `XarrayMask`: align/apply masks to model outputs diff --git a/docs/source/development/offline-era5-fixture-datasets.md b/docs/source/development/offline-era5-fixture-datasets.md new file mode 100644 index 00000000..74018c37 --- /dev/null +++ b/docs/source/development/offline-era5-fixture-datasets.md @@ -0,0 +1,82 @@ +# Offline ERA5 fixture datasets (`*_test` weather configs) + +This document records the **design and implementation plan** for small, committed NetCDF fixtures used in automated tests—without calling the CDS API or relying on `DATASET_ROOT_PATH` downloads. + +## Goals + +- Ship **minimal** ERA5-shaped files in the repository for CI and local testing. +- Expose them via **`load_dataset("…_test")`** so code paths mirror production (`wind_3d_hourly`, `wind_solar_hourly`) while staying **offline**. +- Avoid coupling tests to arbitrary year/month ranges: fixture datasets should use a **fixed catalog** (typically a single file) even if `BaseDataset.__init__` still requires `years` / `months` arguments (those values can be **ignored** for catalog construction in test configs). + +## Non-goals + +- The legacy **`geodata.dataset.Dataset`** (`module=` + `weather_data_config=` dict) is **not** in scope; the plan targets **`load_dataset` + `BaseDataset` subclasses** used by models and current tests. + +## Current fixture layout (repository) + +Fixtures live under **`tests/fixtures/`** so they stay close to pytest and do not inflate the installable package unless explicitly packaged later. + +| Test weather config (planned) | Mirrors production config | On-disk layout under `tests/fixtures/era5/` | +|-------------------------------|---------------------------|---------------------------------------------| +| `wind_3d_hourly_test` | `wind_3d_hourly` (`frequency="daily"`) | `wind_3d_hourly_test/2016/01/01.nc` | +| `wind_solar_hourly_test` | `wind_solar_hourly` (default `frequency="monthly"`) | `wind_solar_hourly_test/2016/01.nc` | + +Production datasets store files under: + +`DATASET_ROOT_PATH / / / …` + +with: + +- **Daily** (3D wind): `…///.nc` +- **Monthly** (wind/solar hourly): `…//.nc` + +The fixture tree **matches those relative paths** so `AtomicDataset.path` resolution stays aligned with the real datasets. + +## Registry and naming + +- Each test variant is a **`BaseDataset` subclass** with `weather_config = "wind_3d_hourly_test"` or `"wind_solar_hourly_test"`. +- Subclasses are registered automatically via `BaseDataset.__init_subclass__` into `geodata.datasets.registry`. +- Callers use **`load_dataset("wind_3d_hourly_test")`** (same pattern as production). + +## Behavioral contract + +### Storage root + +Fixture classes should set **`storage_root`** to the directory that contains the fixture tree for that config—for example, the absolute path to `tests/fixtures/era5/wind_3d_hourly_test` resolved at runtime (repo-relative or via `importlib.resources` if fixtures are ever packaged). + +### Catalog + +Override **`catalog`** so it returns **only** the `AtomicDataset` entries that refer to committed files (commonly **one** file): + +- 3D wind: one daily file, e.g. `(year=2016, month=1, day=1)` → `…/2016/01/01.nc` +- Wind/solar: one monthly file, e.g. `(year=2016, month=1)` → `…/2016/01.nc` + +Constructor arguments **`years` / `months`** may remain required by `BaseDataset.__init__` but **need not drive** the fixture catalog. + +### Download + +- **`download()`** must **not** call CDS: implement as a no-op or raise a clear error if invoked. +- **`_download_file`** should not perform network I/O. + +### Prepared state + +`downloaded` should become **`True`** when fixture files exist (the default `_check_downloaded()` loop over `catalog` is sufficient if paths resolve correctly). + +## Models and `SUPPORTED_WEATHER_DATA_CONFIGS` + +`BaseModel` validates both **`weather_config`** and **`source.downloaded`**. Any model that should run on fixtures must **allow** the `*_test` config names—e.g. extend `SUPPORTED_WEATHER_DATA_CONFIGS` on `WindInterpolationModel`, pvlib-related models, and any other entry points used in tests—to include `wind_3d_hourly_test` / `wind_solar_hourly_test` (or document a single shared alias strategy). + +## Implementation checklist + +1. Add **`ERA5Wind3DHourlyTestDataset`** / **`ERA5WindSolarHourlyTestDataset`** (names may vary) beside the existing ERA5 hourly classes, or in a small `fixture.py` module imported from `era5` packages so subclasses register on import. +2. Wire **`storage_root`** to `tests/fixtures/era5//` (resolve path robustly from the repo root or test layout). +3. Override **`catalog`** to the fixed fixture file(s); ignore user `years`/`months` for catalog purposes (documented). +4. Override **`download`** / **`_download_file`** to prevent CDS usage. +5. Update **`SUPPORTED_WEATHER_DATA_CONFIGS`** on affected models. +6. Add or adjust tests: `load_dataset("…_test")`, assert `downloaded`, **no** `download()`, then run the intended model or pipeline assertion. + +## References (code) + +- Registry: `geodata.datasets._base.BaseDataset.__init_subclass__` +- Paths: `AtomicDataset.path` in `geodata.datasets._base` +- Legacy downloader: `geodata.dataset.Dataset` (separate from this plan) diff --git a/docs/source/development/xarray_mask_workflow.rst b/docs/source/development/xarray_mask_workflow.rst new file mode 100644 index 00000000..929a115f --- /dev/null +++ b/docs/source/development/xarray_mask_workflow.rst @@ -0,0 +1,89 @@ +Xarray masking workflow +========================= + +.. note:: + + **Audience:** contributors and maintainers. This page records the xarray-first + masking implementation phases. For usage, see + :doc:`/mask/xarray_mask_tutorial`. + +This page summarizes the **xarray-first masking** work added alongside the +longer-term plan in :doc:`mask_xarray_migration_plan`. The legacy path based on +``Cutout`` (``add_mask``, ``add_grid_area``, ``mask``) is unchanged for now; the +new pieces let you mask **any** model or analysis output +given as an ``xarray.Dataset`` or ``xarray.DataArray``, without threading mask +logic through model classes. + +What was added +-------------- + +**Phase 0 — behavior freeze (tests only)** + +Offline tests lock in legacy masking behavior so refactors do not silently change +results: + +* Coarsening / alignment of saved mask rasters onto a target grid. +* Grid cell area computation consistent with the cutout-style workflow. +* The structure of outputs from ``Cutout.mask(...)`` (keys, variables, dimensions). +* Selected error paths (missing mask, missing area, invalid mask state). + +**Phase 1 — shared spatial helpers** + +The following helpers now live in ``geodata.mask.spatial`` and are re-used from +``cutout`` (and plotting code where relevant): + +* ``ds_reformat_index`` — normalize coordinates toward ``lat`` / ``lon``. +* ``coarsen`` — align a higher-resolution mask grid to a target grid. +* ``calc_grid_area`` / ``calc_shp_area`` — area utilities used by the masking workflow. + +Public names on ``geodata.cutout`` (e.g. ``coarsen``, ``calc_grid_area``) remain +available as aliases for backward compatibility. + +**Phase 2 — ``XarrayMask``** + +``XarrayMask`` (``from geodata import XarrayMask``) provides: + +* ``from_name`` / ``from_mask`` — load a saved ``Mask`` and align + merged and shape masks to a target ``grid`` (your model output or any dataset + with compatible ``x``/``y`` or ``lat``/``lon`` coordinates). +* ``compute_grid_area`` — per-cell area on the target grid (same idea as cutout + grid area). +* ``attach`` — return a dict of datasets like legacy ``Cutout.mask``: original + variables plus ``mask`` and optional ``area``. +* ``apply`` — return masked data (``mode="where"`` for NaN outside mask, + ``mode="multiply"`` for zero outside mask), optionally with ``area``. + +**Integration pattern (no coupling inside models)** + +Masking is intentionally **not** built into wind, pvlib, or other model ``estimate`` +APIs. The intended usage is: + +1. Run the model and obtain ``output_ds`` (or a ``DataArray`` you wrap in a + one-variable dataset). +2. Build ``XarrayMask.from_name("my_mask", grid=output_ds, mask_dir=...)`` if needed. +3. Call ``attach(output_ds)`` or ``apply(output_ds, ...)`` for analysis. + +See :doc:`/mask/xarray_mask_tutorial` for a step-by-step notebook, and the offline +tests under ``tests/pr/`` (e.g. ``test_xarray_mask.py``, +``test_wind_xarraymask_integration.py``) for concrete examples. + +Package layout note +------------------- + +The repository currently has both: + +* ``src/geodata/mask.py`` — original ``geodata.mask`` implementation (``Mask``, + raster helpers, etc.). +* ``src/geodata/mask/`` — package namespace that re-exports that API **and** + hosts new modules (``spatial.py``, ``xarray_mask.py``). + +Imports like ``from geodata import Mask`` and ``from geodata import XarrayMask`` +continue to work during this transition. + +See also +-------- + +* :doc:`/mask/xarray_mask_tutorial` — step-by-step notebook (offline runnable). +* :doc:`mask_xarray_migration_plan` — full migration phases and deprecation plan. +* :doc:`/legacy/mask_on_cutout` — legacy notebook: masks via ``Cutout``. +* :doc:`/mask/mask_creation_workflow` — building and saving ``Mask`` objects from rasters. diff --git a/docs/source/index.rst b/docs/source/index.rst index fc398a3b..a72c54c0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,31 +17,38 @@ Welcome to Geodata's documentation! quick_start/input_output .. toctree:: - :caption: Dataset Specific Tutorials + :maxdepth: 1 + :caption: Legacy workflow + :hidden: + + legacy/index + +.. toctree:: + :caption: Datasets :maxdepth: 1 :glob: :hidden: - datasets/era5/index - datasets/merra2/index datasets/* .. toctree:: :maxdepth: 1 :caption: Modeling - :glob: :hidden: + modeling/era5_outputs modeling/wind/index - modeling/* + modeling/pvlib/index .. toctree:: :maxdepth: 1 :caption: Mask - :glob: :hidden: - mask/* + mask/mask_creation_workflow + mask/xarray_mask_tutorial + mask/mask_troubleshoot + mask/merge_layer_known_issues .. .. toctree:: .. :maxdepth: 1 @@ -66,6 +73,16 @@ Welcome to Geodata's documentation! .. application/* +.. toctree:: + :maxdepth: 1 + :caption: Development + :hidden: + + development/documentation-organization-plan + development/offline-era5-fixture-datasets + development/mask_xarray_migration_plan + development/xarray_mask_workflow + .. toctree:: :maxdepth: 1 :caption: API Reference diff --git a/docs/source/intro.rst b/docs/source/intro.rst index f29bb772..f4fc47e4 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -7,7 +7,7 @@ engineering, and social science applications. .. figure:: _static/images/geodata_workflow_chart.png :alt: Geodata Workflow - A typical anaylsis workflow with Geodata + A typical analysis workflow with Geodata Motivation ---------- @@ -31,222 +31,130 @@ model inputs. Additionally, with a minimal amount of data consistency checks and metadata information, when one researcher goes through this exercise, everyone benefits. -How To Use +How to use ---------- -Download Datasets -~~~~~~~~~~~~~~~~~ - -Earth system datasets can be large (100+ MB / file with hundreds of -files necessary for a single analysis) and their APIs and file -structures (e.g., daily vs monthly) vary by source. Utilizing xarray and -dask data parallelization, Geodata provides single call download with -API credentials stored locally. Data requests are automatically trimmed -to keep only required variables, significantly reducing bandwidth -requirements and disk usage. +Overview +~~~~~~~~ -Geodata currently supports MERRA-2 and ERA5 reanalysis products and -various GIS file formats (see :doc:`here`). -For example, to evaluate solar PV availability using -`MERRA2 `__ -on 01/01/2011, use the following method call: +The recommended workflow follows four steps: -.. code :: Python +1. **Load and download** a registered dataset with ``load_dataset``. +2. **Run a model** (wind or solar PV) to produce xarray outputs. +3. **Apply a mask** (optional) with ``XarrayMask`` on model output. +4. **Analyze or visualize** the results in xarray, pandas, or with + ``geodata.plot``. - from geodata import Dataset +Geodata supports ERA5 reanalysis through ``load_dataset`` and common GIS +formats (see :doc:`quick_start/input_output`). For dataset setup and configs, +see :doc:`datasets/overview`. MERRA-2 cutout workflows are documented under +:doc:`/legacy/index`. - solar = Dataset( - module="merra2", - years= slice(2011, 2011), - months=slice(1,1), - weather_data_config="slv_radiation_hourly" - ) - solar.get_data() +.. note:: -Extract Cutouts -~~~~~~~~~~~~~~~ + If you rely on the older ``Dataset`` / ``Cutout`` / ``convert`` API, + see :doc:`legacy/workflow`. -Most energy analyses (e.g., energy models, resource assessments, -political economy studies) require time series on subsets of locations -and time periods. Geodata can extract desired variables, time periods, -and geographies from the dataset to a Cutout object. We then call various functions in -``geodata.convert`` module to transform the raw data into analysis-ready -variables with the option to export to CSV or combine with other GIS -datasets through further masking analysis. +Step 1: Load and download a dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -After downloading the required -`MERRA2 `__ -dataset, we create a Cutout object that contains solar irradiance over -China. +Earth system datasets can be large (100+ MB per file, with many files +per analysis). The ``geodata.datasets`` module provides a unified +interface: pick a registered config, instantiate the dataset class, and +download only the variables and time range you need. .. code :: Python - from geodata import Cutout + from geodata.datasets import load_dataset - cutout = Cutout( - name="china-2011-slv-hourly-test", - module="merra2", - weather_data_config="slv_radiation_hourly", - xs=slice(73, 136), - ys=slice(18, 54), - years=slice(2011, 2011), + ds_cls = load_dataset("wind_3d_hourly") + ds = ds_cls( + years=slice(2016, 2016), months=slice(1, 1), + bounds=[-10, 35, 10, 45], # optional bounding box ) - cutout.prepare() - - -Then, we can convert the downward-shortwave, upward-shortwave radiation -flux, and ambient temperature variables from the Cutout data into a PV -generation time-series using the geodata ``convert`` method. Geodata -stores objects internally as an xarray DataArray, which can be easily -converted to a Pandas DataFrame. - -.. code :: Python - from geodata import convert + if not ds.downloaded: + ds.download() - ds_solar = convert.pv(cutout, panel="KANEKA", orientation="latitude_optimal") - ds_solar.to_dataframe(name="pv") + print(ds.downloaded) +Use ``list_datasets()`` to see all registered configs. For ERA5 CDS +credentials and offline test fixtures, see :doc:`datasets/era5` and +:doc:`development/offline-era5-fixture-datasets`. -.. figure:: _static/images/example_output_dataframe.png - :alt: Output DataFrame - :scale: 50% +Step 2: Run a model +~~~~~~~~~~~~~~~~~~~ - Output of the code above +Models operate on downloaded datasets and return **xarray** objects. +Import the model explicitly (models are not re-exported at the top-level +``geodata`` namespace). -We can plot a time series of average PV values for all grid cells on -that day with geodata's visualization method: +**Wind** — interpolate or extrapolate hub-height wind speed and capacity +factor from ERA5 3D wind data: .. code :: Python - from geodata import plot - - plot.time_series(ds_solar) + from geodata.model.wind import WindInterpolationModel -.. figure:: _static/images/visualization/output_12_0.png - :alt: Time-Series Plot + model = WindInterpolationModel(ds) + model.prepare() + wind_speed = model.estimate(height=100.0) - Visualization of the average PV values over time +See :doc:`modeling/wind/index` for wind interpolation and turbine capacity factor, and +turbine capacity-factor details. -We can also visualize the average solar PV for every two hours on this -day through an animation: +**Solar PV** — estimate AC power and capacity factor with pvlib-backed +models on ERA5 wind/solar hourly data: .. code :: Python - import geopandas as gpdø - - from geodata import plot - - prov_shapes = gpd.read_file(prov_shapes_path) - geodata.plot.heatmap_animation( - ds_solar, - cmap="Wistia", - time_factor=2, - shape=prov_shapes, - shape_width=0.25, - shape_color="navy", - ) - + from geodata.datasets import load_dataset + from geodata.model.pvlib import Pvlib -.. figure:: _static/images/visualization/pv_animation.gif - :alt: animation + solar_cls = load_dataset("wind_solar_hourly") + solar_ds = solar_cls(years=slice(2016, 2016), months=slice(1, 1)) + if not solar_ds.downloaded: + solar_ds.download() - Animated Result + pv_model = Pvlib(solar_ds) + # configure pv_system and model config — see modeling/pvlib/index + cf = pv_model.estimate(years=slice(2016, 2016), months=slice(1, 1)) -Masking -~~~~~~~ +See :doc:`modeling/pvlib/index` for full PV system and ModelChain setup. -Geographic masks help filter datasets for specific analyses. Geodata is -able to process GIS datasets and extract cutouts over specified -geographies. Built off the open-source binary libraries GDAL, GEOS, and -PROJ, and Python libraries rasterio and shapely, the Mask module imports -rasters and shapefiles, edits them as mask layers, merges and flattens -multiple layers together, and extracts subsetted cutout data from merged -masks and shapefiles. +Step 3: Apply a mask (optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For example, within Geodata the user can load the `MODIS land use -dataset `__, -the `elevation -dataset `__, -and `environmental protected -shapes `__, filter these -according to solar energy suitability criteria, and merge into a single -binary siting mask, where values of 0 represent the unsuitable area, and -values of 1 represent the suitable area. Masks can be saved locally for -later use. - -Geodata automatically reprojects GIS data in different coordinate -reference systems into degree coordinates for processing. Common -manipulations include cropping, filtering on categorical values, -filtering on thresholds, excluding small contiguous areas, and filtering -by shape buffers. One multi-purpose plotting function (``mask.show``) -supports visualizing the mask including relevant shape boundaries. - -For example, Geodata can create a binary mask of wind energy suitability -in China based on the above GIS inputs. +Mask **creation** uses ``geodata.Mask`` (see +:doc:`mask/mask_creation_workflow`). To apply a saved mask to model +output without a ``Cutout``, use ``XarrayMask``: .. code :: Python - import geopandas as gpd - - from geodata import mask - - china = mask.Mask("China") - china.add_layer(layer_path={"modis": modis_path, "elevation": elevation_path}) - - protected_area_shapes = gpd.read_file(protected_area_shapes_path) - china.add_shape_layer( - protected_area_shapes["geometry"].to_dict(), - reference_layer="elevation", - combine_name="protected", - buffer=20, - ) - - china.filter_layer( - "modis", binarize=True, values=[6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17] - ) - china.filter_layer("elevation", binarize=True, max_bound=4000) - china.merge_layer(trim=True) + from geodata import XarrayMask - china_prov_shapes = gpd.read_file(china_prov_shapes_path) - mask.show(china.merged_mask, shape=china_prov_shapes["geometry"], title="Merged Mask") + xmask = XarrayMask.from_name("my_mask", grid=wind_speed) + masked = xmask.apply(wind_speed, mode="where") - china.save_mask() +See :doc:`mask/xarray_mask_tutorial` for ``attach``, ``apply``, and +grid-area weighting. -.. figure:: _static/images/mask_workflow.png - :alt: mask workflow - - Visualization of Mask Workflow +Step 4: Visualize +~~~~~~~~~~~~~~~~~ -In the final step, we apply the Mask object to the Cutout. Geodata -automatically coarsens the (typically) high-resolution Mask into the -same resolution as the Cutout, adding fractions of the coarse cells -covered by the Mask and areas calculated via an equal-area projection. +Plotting works on any xarray object returned by a model: .. code :: Python - ds_cutout = convert.pv( - cutout, panel="KANEKA", orientation="latitude_optimal" - ).to_dataset(name="solar") - - cutout.add_mask("china") - cutout.add_grid_area() - ds_mask = cutout.mask(dataset=ds_cutout)["merged_mask"] - - weighted_mean_pv_series = ( - (ds_mask["solar"] * ds_mask["mask"] * ds_mask["area"]).sum(axis=1).sum(axis=1) - ) / (ds_mask["mask"] * ds_mask["area"]).sum() - - plt.plot(weighted_mean_pv_series) - + from geodata import plot -.. figure:: _static/images/mask_cutout_workflow.png - :alt: Mask-Cutout Workflow + plot.time_series(wind_speed) - Mask-Cutout Workflow +See :doc:`visualization/visualization` for heatmaps and animations. What's next? ============ -To further explore the capabilities of Geodata, check out the table of contents on the left! +Use the table of contents on the left to go deeper into datasets, +modeling, masking, and the API reference. diff --git a/docs/source/legacy/index.rst b/docs/source/legacy/index.rst new file mode 100644 index 00000000..1a57f52a --- /dev/null +++ b/docs/source/legacy/index.rst @@ -0,0 +1,25 @@ +Legacy workflow +=============== + +The pages below document the original Geodata API built around +``Dataset``, ``Cutout``, ``geodata.convert``, and Cutout-based masking, +including MERRA2 download and cutout tutorials. + +.. note:: + + This path is **not** part of the current tested workflow + (``load_dataset`` → models → ``XarrayMask``). It remains available for + existing analyses and reference. + +For the recommended path, see the :doc:`documentation homepage `. + +.. toctree:: + :maxdepth: 1 + + workflow + mask_on_cutout + merra2/index + merra2/merra2_download + merra2/merra2_outputs + merra2/merra2 + wind_extrapolation diff --git a/docs/source/mask/mask_on_cutout.ipynb b/docs/source/legacy/mask_on_cutout.ipynb similarity index 100% rename from docs/source/mask/mask_on_cutout.ipynb rename to docs/source/legacy/mask_on_cutout.ipynb diff --git a/docs/source/datasets/merra2/index.md b/docs/source/legacy/merra2/index.md similarity index 77% rename from docs/source/datasets/merra2/index.md rename to docs/source/legacy/merra2/index.md index fb020ff4..d571067a 100644 --- a/docs/source/datasets/merra2/index.md +++ b/docs/source/legacy/merra2/index.md @@ -1,5 +1,11 @@ # MERRA2 Related Tutorials +```{note} +**Legacy documentation.** These tutorials use the older ``Dataset`` / ``Cutout`` API and are +not part of the current tested workflow. For the recommended ERA5 path, see +[Dataset module overview](../../datasets/overview.rst) and [ERA5 setup](../../datasets/era5.rst). +``` + This page explains how you can setup access MERRA2 data from NASA's [GES DISC](https://disc.gsfc.nasa.gov/). ## Creating an Earthdata Login Profile and Approving the GES DISC App @@ -41,9 +47,12 @@ For Windows, open Notepad and enter the following line in a new document, making Save the file to `C:\Users\\.netrc` -## What' next? +## What's next? Now that you have configured your Earthdata Login credentials, you have successfully set up access to the MERRA-2 data. -Please subsequently refer to the [general documentation on datasets](../overview.rst) -for more information on how to download ERA5-based datasets using the `geodata` -package. + +* [Download MERRA2 data and create cutouts](merra2_download.md) +* [MERRA2 outputs via `convert`](merra2_outputs.md) +* [MERRA2 workflow notebook](merra2.ipynb) + +For the current ERA5 + `load_dataset` workflow, see [Dataset module overview](../../datasets/overview.rst). diff --git a/docs/source/datasets/merra2/merra2.ipynb b/docs/source/legacy/merra2/merra2.ipynb similarity index 100% rename from docs/source/datasets/merra2/merra2.ipynb rename to docs/source/legacy/merra2/merra2.ipynb diff --git a/docs/source/datasets/merra2/merra2_download.md b/docs/source/legacy/merra2/merra2_download.md similarity index 100% rename from docs/source/datasets/merra2/merra2_download.md rename to docs/source/legacy/merra2/merra2_download.md diff --git a/docs/source/datasets/merra2/merra2_outputs.md b/docs/source/legacy/merra2/merra2_outputs.md similarity index 100% rename from docs/source/datasets/merra2/merra2_outputs.md rename to docs/source/legacy/merra2/merra2_outputs.md diff --git a/docs/source/legacy/wind_extrapolation.rst b/docs/source/legacy/wind_extrapolation.rst new file mode 100644 index 00000000..81dab3be --- /dev/null +++ b/docs/source/legacy/wind_extrapolation.rst @@ -0,0 +1,102 @@ +Wind extrapolation (legacy) +=========================== + +.. note:: + + **Legacy / untested in CI.** ``WindExtrapolationModel`` only supports the + ``slv_flux_hourly`` weather config (MERRA-2 via ``load_dataset``). It is **not** + part of the current ERA5 workflow documented on the homepage. For ERA5 wind, use + :doc:`/modeling/wind/interpolation` instead. + +For the recommended modern path, see the :doc:`documentation homepage `. + +Tutorial: Estimate Wind Speed with Extrapolation +------------------------------------------------ + +In this tutorial, we will learn how to estimate wind speed using the extrapolation model +from the geodata library. + +.. warning:: + + Extrapolation requires a dataset with wind speed at **multiple** heights. In Geodata, + only ``slv_flux_hourly`` (MERRA-2) is registered for ``WindExtrapolationModel``. + Using any other ``weather_config`` raises ``ValueError``. + +Step 1: Import the necessary libraries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: Python + + import xarray as xr + + from geodata.datasets import load_dataset + from geodata.model.wind import WindExtrapolationModel + + +Step 2: Load the dataset +~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the MERRA-2 ``slv_flux_hourly`` config (not ERA5): + +.. code:: Python + + ds_cls = load_dataset("slv_flux_hourly") + ds = ds_cls( + years=slice(2006, 2006), + months=slice(1, 1), + bounds=[-10, 35, 10, 45], + ) + + if not ds.downloaded: + ds.download() + + print(ds.downloaded) + + +Step 3: Compute extrapolation parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: Python + + model = WindExtrapolationModel(ds) + model.prepare() + +Prepared coefficients are stored under ``GEODATA_ROOT/models/`` (see +:doc:`/modeling/wind/index` — **Preparing the model** for ``prepare`` / ``prepared`` / +``force``). + +Step 4: Estimate using the extrapolation model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: Python + + estimated_wind_speed = model.estimate( + height=60, + years=slice(2006, 2006), + months=slice(1, 1), + ) + +Step 5: Estimate wind turbine capacity factor +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See :doc:`/modeling/wind/interpolation` Step 5 — **Understanding the output** for what +``cf`` means. Example: + +.. code:: Python + + estimated_cf = model.estimate( + turbine="Vestas_V112_3MW", + years=slice(2006, 2006), + months=slice(1, 1), + ) + + +How the Extrapolation Model Works +--------------------------------- + +The model calculates hub height wind speed from MERRA2 surface and low-level winds, +extrapolating variables in MERRA's tavg1_2d_slv_Nx collection (2 m, 10 m, 50 m winds, +displacement height, lowest model level winds, etc.). + +The hub height wind speed uses a log-profile fit (see the original tutorial in the +repository history for the full equations). diff --git a/docs/source/legacy/workflow.rst b/docs/source/legacy/workflow.rst new file mode 100644 index 00000000..476fe909 --- /dev/null +++ b/docs/source/legacy/workflow.rst @@ -0,0 +1,227 @@ +Legacy workflow +=============== + +.. note:: + + This page documents the original ``Dataset`` → ``Cutout`` → ``convert`` workflow. + For the current recommended path, see the :doc:`documentation homepage `. + +How To Use +---------- + +Download Datasets +~~~~~~~~~~~~~~~~~ + +Earth system datasets can be large (100+ MB / file with hundreds of +files necessary for a single analysis) and their APIs and file +structures (e.g., daily vs monthly) vary by source. Utilizing xarray and +dask data parallelization, Geodata provides single call download with +API credentials stored locally. Data requests are automatically trimmed +to keep only required variables, significantly reducing bandwidth +requirements and disk usage. + +Geodata currently supports MERRA-2 and ERA5 reanalysis products and +various GIS file formats (see :doc:`here `). + +**Note**: +If you are exploring ERA5 data with wind or pvlib model, please refer to the :doc:`/modeling/wind/index` and :doc:`/modeling/pvlib/index` pages for more details. +As they are following the dataset module to download data, not the following legacy code. + +For example, to evaluate solar PV availability using +`MERRA2 `__ +on 01/01/2011, use the following method call: + +.. code :: Python + + from geodata import Dataset + + solar = Dataset( + module="merra2", + years= slice(2011, 2011), + months=slice(1,1), + weather_data_config="slv_radiation_hourly" + ) + solar.get_data() + +Extract Cutouts +~~~~~~~~~~~~~~~ + +Most energy analyses (e.g., energy models, resource assessments, +political economy studies) require time series on subsets of locations +and time periods. Geodata can extract desired variables, time periods, +and geographies from the dataset to a Cutout object. We then call various functions in +``geodata.convert`` module to transform the raw data into analysis-ready +variables with the option to export to CSV or combine with other GIS +datasets through further masking analysis. + +After downloading the required +`MERRA2 `__ +dataset, we create a Cutout object that contains solar irradiance over +China. + +.. code :: Python + + from geodata import Cutout + + cutout = Cutout( + name="china-2011-slv-hourly-test", + module="merra2", + weather_data_config="slv_radiation_hourly", + xs=slice(73, 136), + ys=slice(18, 54), + years=slice(2011, 2011), + months=slice(1, 1), + ) + cutout.prepare() + + +Then, we can convert the downward-shortwave, upward-shortwave radiation +flux, and ambient temperature variables from the Cutout data into a PV +generation time-series using the geodata ``convert`` method. Geodata +stores objects internally as an xarray DataArray, which can be easily +converted to a Pandas DataFrame. + +.. code :: Python + + from geodata import convert + + ds_solar = convert.pv(cutout, panel="KANEKA", orientation="latitude_optimal") + ds_solar.to_dataframe(name="pv") + + +.. figure:: ../_static/images/example_output_dataframe.png + :alt: Output DataFrame + :scale: 50% + + Output of the code above + +We can plot a time series of average PV values for all grid cells on +that day with geodata's visualization method: + +.. code :: Python + + from geodata import plot + + plot.time_series(ds_solar) + +.. figure:: ../_static/images/visualization/output_12_0.png + :alt: Time-Series Plot + + Visualization of the average PV values over time + +We can also visualize the average solar PV for every two hours on this +day through an animation: + +.. code :: Python + + import geopandas as gpdø + + from geodata import plot + + prov_shapes = gpd.read_file(prov_shapes_path) + geodata.plot.heatmap_animation( + ds_solar, + cmap="Wistia", + time_factor=2, + shape=prov_shapes, + shape_width=0.25, + shape_color="navy", + ) + + +.. figure:: ../_static/images/visualization/pv_animation.gif + :alt: animation + + Animated Result + +Masking +~~~~~~~ + +Geographic masks help filter datasets for specific analyses. Geodata is +able to process GIS datasets and extract cutouts over specified +geographies. Built off the open-source binary libraries GDAL, GEOS, and +PROJ, and Python libraries rasterio and shapely, the Mask module imports +rasters and shapefiles, edits them as mask layers, merges and flattens +multiple layers together, and extracts subsetted cutout data from merged +masks and shapefiles. + +For example, within Geodata the user can load the `MODIS land use +dataset `__, +the `elevation +dataset `__, +and `environmental protected +shapes `__, filter these +according to solar energy suitability criteria, and merge into a single +binary siting mask, where values of 0 represent the unsuitable area, and +values of 1 represent the suitable area. Masks can be saved locally for +later use. + +Geodata automatically reprojects GIS data in different coordinate +reference systems into degree coordinates for processing. Common +manipulations include cropping, filtering on categorical values, +filtering on thresholds, excluding small contiguous areas, and filtering +by shape buffers. One multi-purpose plotting function (``mask.show``) +supports visualizing the mask including relevant shape boundaries. + +For example, Geodata can create a binary mask of wind energy suitability +in China based on the above GIS inputs. + +.. code :: Python + + import geopandas as gpd + + from geodata import mask + + china = mask.Mask("China") + china.add_layer(layer_path={"modis": modis_path, "elevation": elevation_path}) + + protected_area_shapes = gpd.read_file(protected_area_shapes_path) + china.add_shape_layer( + protected_area_shapes["geometry"].to_dict(), + reference_layer="elevation", + combine_name="protected", + buffer=20, + ) + + china.filter_layer( + "modis", binarize=True, values=[6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17] + ) + china.filter_layer("elevation", binarize=True, max_bound=4000) + china.merge_layer(trim=True) + + china_prov_shapes = gpd.read_file(china_prov_shapes_path) + mask.show(china.merged_mask, shape=china_prov_shapes["geometry"], title="Merged Mask") + + china.save_mask() + +.. figure:: ../_static/images/mask_workflow.png + :alt: mask workflow + + Visualization of Mask Workflow + +In the final step, we apply the Mask object to the Cutout. Geodata +automatically coarsens the (typically) high-resolution Mask into the +same resolution as the Cutout, adding fractions of the coarse cells +covered by the Mask and areas calculated via an equal-area projection. + +.. code :: Python + + ds_cutout = convert.pv( + cutout, panel="KANEKA", orientation="latitude_optimal" + ).to_dataset(name="solar") + + cutout.add_mask("china") + cutout.add_grid_area() + ds_mask = cutout.mask(dataset=ds_cutout)["merged_mask"] + + weighted_mean_pv_series = ( + (ds_mask["solar"] * ds_mask["mask"] * ds_mask["area"]).sum(axis=1).sum(axis=1) + ) / (ds_mask["mask"] * ds_mask["area"]).sum() + + plt.plot(weighted_mean_pv_series) + + +.. figure:: ../_static/images/mask_cutout_workflow.png + :alt: Mask-Cutout Workflow + + Mask-Cutout Workflow diff --git a/docs/source/mask/mask_creation_workflow.ipynb b/docs/source/mask/mask_creation_workflow.ipynb index 636f6a1c..7d532403 100644 --- a/docs/source/mask/mask_creation_workflow.ipynb +++ b/docs/source/mask/mask_creation_workflow.ipynb @@ -1,1151 +1,1151 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tutorial: Typical Mask Workflow" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Introduction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Geodata is able to process geospatial data to extract cutouts over specified geographies. Built off the [rasterio library](https://rasterio.readthedocs.io/en/latest/quickstart.html), the **mask** module imports rasters and shapefiles, merges and flattens multiple layers together, and extracts subsetted cutout data from merged masks and shapefiles.\n", - "\n", - "Functionalities explored in this notebook:\n", - "\n", - "- [Creating a mask object, adding and manipulating layers](#mask-creation--adding-and-manipulating-layers)\n", - "- [Opening a shapefile and adding shape features as layers](#adding-shape-features-as-a-layer)\n", - "- [Merging and flattening layers](#merging-and-flattening-layers)\n", - "- [Eliminate small contiguous areas](#eliminate-small-contiguous-areas)\n", - "- [Extracting shapes from mask](#extracting-shapes-from-masks)\n", - "- [Saving and loading masks](#saving-and-loading-masks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To start, import the geodata package and required libraries. We can also import the `geodata.mask.show()` method for simplicity of its use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import geopandas as gpd\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "import geodata\n", - "from geodata.mask import show" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Additionally, we use [cartopy](https://scitools.org.uk/cartopy/docs/latest/tutorials/using_the_shapereader.html#cartopy.io.shapereader.Reader) to download some common administrative region shapes, but user-provided shapefiles will also work:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import cartopy.io.shapereader as shpreader" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Shapefiles and Rasters\n", - "\n", - "We will use the following geotiff and shape files for this demo:\n", - "\n", - "\n", - "- `china_modis.tif`\n", - "\n", - " We downloaded the MODIS land cover data, which uses satellite remote sensing data to estimate the land use type on an annual basis. See: [EarthData_MCD12Q1](https://lpdaac.usgs.gov/products/mcd12q1v006/).\n", - "\n", - " We will use the IGBP classification ('LC_Type1') which has 17 different land use characterizations (the corresponding data thus takes values from 1.0 to 17.0).\n", - " All the \"Bands\" are listed here: [Google_earth_engine_MODIS_006_MCD12Q1](https://developers.google.com/earth-engine/datasets/catalog/MODIS_006_MCD12Q1#bands)\n", - "\n", - "- `china_elevation.tif` and `china_slope.tif`\n", - "\n", - " These two rasters are based on the elevation map from: [Google_earth_engine_MODIS_CGIAR_SRTM90_V4](https://developers.google.com/earth-engine/datasets/catalog/CGIAR_SRTM90_V4?hl=en). Slope was computed in degrees using the 4-connected neighbors of each pixel. \n", - "\n", - "\n", - "- `UNEP_WDPA_China` Shapefiles\n", - "\n", - " We downloaded the environmental protected area for China from: [ProtectedPlanet_China](https://www.protectedplanet.net/country/CHN). These shapefiles are distributed among 3 subfolders upon successful download and decompression due to the large size. We will create path variables for all three subfolders and we will only take the polygon shapes.\n", - "\n", - " Alternatively, We can also retrieve the environmental protected area from Google Earth Engine: [Google_earth_engine_WCMC_WDPA](https://developers.google.com/earth-engine/datasets/catalog/WCMC_WDPA_current_polygons). The shapefile will contain the protected shapes from entire world (and the size is slightly over 1 GB), and additional data cleaning will be necessary if the user wants just the shapes within China. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "modis_path = \"data/china_modis.tif\"\n", - "elevation_path = \"data/china_elevation.tif\"\n", - "slope_path = \"data/china_slope.tif\"\n", - "\n", - "wdpa_shape_path_0 = \"data/shapefiles/0/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"\n", - "wdpa_shape_path_1 = \"data/shapefiles/1/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"\n", - "wdpa_shape_path_2 = \"data/shapefiles/2/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let us get province shapes from `cartopy` and save the path as `prov_path`. This can also be the path to user-supplied shape files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prov_path = shpreader.natural_earth(\n", - " resolution=\"10m\", category=\"cultural\", name=\"admin_1_states_provinces\"\n", - ")\n", - "prov_path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load the shapes contained in path `prov_path` using the `geopandas` library." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "all_shapes = gpd.read_file(prov_path, encoding=\"utf-8\")\n", - "all_shapes.head(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "GeoPandas data filtering with GeoDataFrame is identical to pandas. Let us select all the rows that contains shape within China." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china_shapes = all_shapes[all_shapes[\"admin\"] == \"China\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, to load the WDPA environmental protected shapefiles as a layer in the china mask, we will use the GeoPandas library. `gpd.read_file()` will return a GeoPandas dataframe including shape attributes and geometry given the file path. Like Pandas, we can read multiple dataframes and concat them together. In the code below, we will create one GeoPandas dataframe from three paths that we have for the Chinese environmental protected shapes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "wdpa_shapes = pd.concat([\n", - " gpd.read_file(wdpa_shape_path_0),\n", - " gpd.read_file(wdpa_shape_path_1),\n", - " gpd.read_file(wdpa_shape_path_2)\n", - "])\n", - "wdpa_shapes.head(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Mask Creation & Adding and Manipulating Layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The mask object consists of multiple layers and manipulations performed on them. To add a layer, the four methods below perform same functions. A user may add a layer to the mask by specifying paths when a new instance is created, or use the `add_layer` method. We will add the following two files: `china_elevation.tif`, and `china_modis.tif` to the `China` mask, and name them `elevation` and `modis` layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# Method 1: Initialize one layer, add one layer\n", - "china = geodata.Mask(\"China\", layer_path=elevation_path)\n", - "china.rename_layer(\"china_elevation\", \"elevation\")\n", - "china.add_layer(modis_path, layer_name=\"modis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Method 2: Initialize empty, add two layers using dict\n", - "china = geodata.Mask(\"China\")\n", - "china.add_layer(layer_path={\"elevation\": elevation_path, \"modis\": modis_path})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Method 3: Initalize with two layers passed as list\n", - "china = geodata.Mask(\n", - " \"China\", layer_path=[elevation_path, modis_path], layer_name=[\"elevation\", \"modis\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Method 4: Initialize with two layers passed as dict\n", - "china = geodata.Mask(\n", - " \"China\", layer_path={\"elevation\": elevation_path, \"modis\": modis_path}\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Display the mask object in the jupyter notebook:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Each mask object has several attributes:\n", - "- `layers`: a dictionary of name (key) - rasterio file opener (values). The <\\open DatasetReader> can be the input for many other mask methods for the module. \n", - "- `merged_mask`: the merged and flatten mask of its layers, the merged raster from `layers`\n", - "- `shape_mask`: similar to the `layers` attribute, but a dictionary of extracted shapes from the merged mask by default. Users may also extracted shape masks from specified layers in `self.layers`.\n", - "- `saved`: whether this mask object has been saved locally.\n", - "- `mask_dir`: the directory to save the mask object, by default it should be the mask dir in config.py." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Show the `slope` layer in mask `china`. The `show` method will always try to show the proper latitude and longitude, unless we call it `show(layer, lat_lon = False)`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.layers[\"elevation\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "show(china.layers[\"elevation\"], title=\"Elevation of China in meters\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Some useful methods to examine the layers**\n", - "\n", - "- `china.get_res()`: get resolution of each layer, in lat-lon coordinates\n", - "- `china.get_res(product = True)`: get grid cell size, in product of lat-lon coordinate differences\n", - "- `china.get_bounds()`: get bounds, in lat-lon coordinates" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.get_bounds()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the modis layer has a very different bounding box then the slope layer in lat-lon coordinate system. This is because the modis layer was converted to the lat-lon CRS from a different CRS when it was added to the object. The following section will explore CRS conversion." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### CRS conversion, trimming, and cropping (Optional)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Method `open_tif` can open a layer without adding it to the layer, this allows us to visualize it before-hand. It is a good practice to close the raster after opening it to avoid writing permission conflict issues. Closing the raster below does not involve any layer operation associated with the mask object. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "modis_opener = geodata.mask.open_tif(modis_path, show_raster=True)\n", - "modis_opener.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can use `remove_layer` method to remove a layer to mask `china`. This method will properly close the raster file, because the raster file would remain open after being added to the mask." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.remove_layer(\"modis\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `add_layer` method incorporates coordinate reference system (CRS) conversion to lat-lon (EPSG:4326), if necessary. Note that this method will overwrite the layer by default, if it is in the object already, unless the user specifies `replace=False`. \n", - "\n", - "The method will automatically trim the all-zero columns/rows. By default, the paramater `trim` is set to `True`. If we do not set it to True, we might generate a converted raster with new CRS but many all-zero columns and rows." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.add_layer(modis_path, \"modis\", trim=False)\n", - "show(china.layers[\"modis\"], title=\"China Modis CRS converted (No trimming)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also crop a raster/layer with user-defined dimensions: method `crop_layer` can take either starting indices of top/left, ending indices of right/bottom, or coordinates values in lat/long to trim the raster.\n", - "\n", - "The difference between `crop_layer` and `trim_layer` is that `crop_layer` must take in user specified range to crop the raster, and `trim_layer` would remove the all zero rows and columns automatically for a raster. So that if the user do not know which index to start and end to remove the empty rows/columns, `trim_raster` is better.\n", - "\n", - "The method `crop_raster` (`geodata.mask.crop_raster`) is similar to `crop_layer` but can take a layer name as input, so that the user does not need to add a raster as a layer to call that method. (Similar method: `trim_layer`/`trim_raster`, `binarize_layer`/`binarize_raster`)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.crop_layer(\"modis\", bounds=(73, 17, 135, 54))\n", - "show(china.layers[\"modis\"], title=\"China Modis Layer Cropped\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This performs the same function by passing the layer to `crop_raster`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.layers[\"modis\"] = geodata.mask.crop_raster(\n", - " china.layers[\"modis\"], (73, 17, 135, 54)\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Filter a layer" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The mask module also supports filtering a layer based on list of categorical values, a minimum (lower) boundary, or maximum (upper) boundary.\n", - "\n", - "In the `filter_raster` method, a user may specify any of the `value` (the list of numberic values in the raster array to be selected), `max_bound`, and `min_bound` parameters to selected desired values. If the parameter `binarize` is False (by default), the method will return the original values of the raster that satisfy the conditions, otherwise the method will return 1 for the values that satisfy the conditions and 0 elsewhere." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Select Categorical Values from MODIS Layer\n", - "\n", - "Since the modis layer has 17 distinct values for different land use types, we want to create a layer of binary values, indicating unavailable land as 0, and available land as 1.\n", - "\n", - "We wish to create a mask where :\n", - "\n", - "- all forested areas (values 1-5) are 0 (i.e., unsuitable)\n", - "- all urban areas (13) are 0\n", - "- all others are 1\n", - "\n", - "\n", - "Let us use method `filter_raster` to create a layer of `modis_filtered` binary mask, where 1, 2, 3, 4, 5, and 13 will be unavailable land assigned 0 and the rest of the values will be 1 (available).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "avail_values = list(set(range(1, 18)) - {1, 2, 3, 4, 5, 13})\n", - "avail_values" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.layers[\"modis_filtered\"] = geodata.mask.filter_raster(\n", - " china.layers[\"modis\"], binarize=True, values=avail_values\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "china.remove_layer(\"modis\")\n", - "show(china.layers[\"modis_filtered\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Filter elevation layer\n", - "\n", - "Because we cannot build renewable energy in areas with high elevation, let us set the constraint from the `elevation` layer, by using elevation < 4000m at 1 and other areas as 0. The result layer `elevation_filtered` will have only 1 and 0 as unique values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.filter_layer(\n", - " \"elevation\", dest_layer_name=\"elevation_filtered\", max_bound=4000, binarize=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.remove_layer(\"elevation\")\n", - "show(china.layers[\"elevation_filtered\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Filter Slope Layer\n", - "\n", - "We also cannot build renewable energy in area with large slopes, so let us set another constraint from the `slope` layer from the slope tif file, by using slope < 20 degree at 1 and else as 0. The result layer `slope_filtered` will have only 1 and 0 as unique values." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, add the slope raster to the china mask." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.add_layer(slope_path, layer_name=\"slope\")\n", - "show(china.layers[\"slope\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Filter the raster, delete the old slope layer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.filter_layer(\n", - " \"slope\", dest_layer_name=\"slope_filtered\", max_bound=20, binarize=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "china.remove_layer(\"slope\")\n", - "show(china.layers[\"slope_filtered\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Additional Visualization Options" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can plot the provinces on a selected layer by taking `shape` input in the `show()` method. Here, we will use the `china_shapes` that we obtained from `all_shape`. Its `geometry` column is a Series of shapes (shapely.geometry or MultiPolygon) for Chinese provinces." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "show(china.layers[\"modis_filtered\"], shape=china_shapes[\"geometry\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Adding Shape Features as a Layer" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Recall that we have previously loaded the environmental protected shapes of China in a GeoPandas dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "len(wdpa_shapes)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The three shapefiles have 78 features altogether, but we want to add all the features to one new layer instead of 78 new layers. The input shape should be a python dictionary, where there is a key for each unique shape. Also, in the `add_shape_layer` method, we will specify a `combine_name` to combine the features into one layer in this case, since we want the mask to have just one more layers, not 78 more layers.\n", - "\n", - "When adding a shapefile, we must specify the dimensions. We will also use `reference layer = 'slope_filtered'` so the new shape layer will have the same dimension with the `slope_filtered` layer. If the mask is empty and does not contain any layer, the user will have to specify the `resolution` parameter for the raster layer dimension.\n", - "\n", - "By default, this method will have paramater `exclude` that defaults to `False`. When it is true, area inside the shape is 0. When it is false, area inside the shape is 1. In this use case, however, we want 0 for area inside of the shape as they are environmental protected areas to exclude. We can just use the default method call." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.add_shape_layer(\n", - " wdpa_shapes[\"geometry\"].to_dict(),\n", - " reference_layer=\"slope_filtered\",\n", - " combine_name=\"protected\",\n", - ")\n", - "show(\n", - " china.layers[\"protected\"],\n", - " title=\"WDPA Protected area shape features as a new layer\",\n", - " grid=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also use the parameter `buffer` in `add_shape_layer` method to create an approximate representation of all locations within a given (perpindicular) distance of the shape object. The units for the buffer are given in kilometers.\n", - "\n", - "Note that since the units of the original shape are in lat-lon coordinates, when we add the buffer, we will need to have a CRS that has meter as unit. The program will convert the shapes to that CRS, add the buffer around shapes, then convert it back to the lat-lon CRS system. By default, we used \"EPSG:6933\", an equal area projection CRS to add buffer in kilometer. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "km_buffer = 20\n", - "\n", - "china.add_shape_layer(\n", - " wdpa_shapes[\"geometry\"].to_dict(),\n", - " reference_layer=\"slope_filtered\",\n", - " combine_name=\"protected_with_buffer\",\n", - " buffer=km_buffer,\n", - ")\n", - "\n", - "show(\n", - " china.layers[\"protected_with_buffer\"],\n", - " title=f\"WDPA Protected area shape with {km_buffer}km buffer\",\n", - " grid=True,\n", - ")\n", - "\n", - "china.remove_layer(\"protected_with_buffer\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Merging and Flattening Layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In order to combine all layers into one, we use the `merge_layer` method which creates a new layer called `merged_mask`. This merges multiple layers together and flattens them using either **and** (default) or **sum** method, saving the result to `self.merged_mask` by default. Geospatial bounds and resolution of the output layer are in the units of the input file coordinate reference system, but by default, we will use the resolution of the layer with the best (finest) resolution for the output bounds/resolution, unless a reference layer is provided. In this case, the resolution of the merged_mask is the same with the `modis_filtered` layer. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.get_res()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merge_layer(attribute_save=False, show_raster=False).res" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Binary `AND` Method" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By default, the `merge_layer` method will use a binary 'and' method: for each grid cell, if any of the n layers are 0, then the returned `self.merged_layer` will also have 0 at that location. In other words, if all the layers indicate that a land is available (!=0), the merged result will have value 1.\n", - "\n", - "`merge_layer` may also take in an optional parameter `layers`, which is a list of layer names stored in the object, if the user does not wish to merge all layers in the object. If the user does not want to save the result to the `merged_mask` attribute, the user can specify `attribute_save = False`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# merge and plot only, do not save\n", - "china.merge_layer(attribute_save=False, layers=[\"slope_filtered\", \"modis_filtered\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Try again with the `reference_layer` parameter:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merge_layer(\n", - " layers=[\"elevation_filtered\", \"modis_filtered\"],\n", - " reference_layer=\"elevation_filtered\",\n", - " show_raster=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The result of the `merged_mask` method is saved to `china.merged_mask` with the same resolution as the reference layer, in this case `elevation_filtered`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merged_mask.res" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For the purpose of this demonstration, we will select the `AND` method for the final merged_mask. We can also trim the border of the merged mask since the 4 layers have different boundaries. We can set the parameter `trim = True`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merge_layer(trim=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### `SUM` Method" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The sum method will add up the values from all the layers using weights. When there is no weight dict provided, all the layers for merging will have weights of 1 by default.\n", - "\n", - "Note: since we are not using the sum method to proceed to the following sections, we will keep `attribute_save = False` to prevent this method from overwriting the mask we have previously created above." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merge_layer(method=\"sum\", attribute_save=False, trim=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This distribution is completely arbitrary for the purpose of demonstration of the module: (Note: The weights do not need to have a total of 1)\n", - "- elevation_filtered: 0.15, slope_filtered: 0.1, modis_filtered: 0.3, protected: 0.45\n", - "\n", - "We will write the result to a new variable `customized_merged_layer` for continuing processing." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "customized_merged_layer = china.merge_layer(\n", - " method=\"sum\",\n", - " weights={\n", - " \"elevation_filtered\": 0.15,\n", - " \"slope_filtered\": 0.1,\n", - " \"modis_filtered\": 0.3,\n", - " \"protected\": 0.45,\n", - " },\n", - " attribute_save=False,\n", - " trim=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the continuous value created by `merged_mask` represents a suitability metric, we could set a minimum value of 0.8 to be considered \"suitable\" (or 1). We then apply the `filter_raster` method on the merged layer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "customized_merged_layer = geodata.mask.filter_raster(\n", - " customized_merged_layer, min_bound=0.8, binarize=True\n", - ")\n", - "show(customized_merged_layer)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Eliminate Small Contiguous Areas" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using the above methods, we might end up with many small contiguous areas that are marked suitable but surrounded by an unsuitable region. We may want to exclude such regions from renewable energy development. The `filter_area` method will remove the small contiguous suitable regions by transforming the merged mask raster to polygons/shapes, calculating the area of each polygon, and filtering out polygons that are smaller than a given threshold. Units are given in kilometer-squared (km$^2$).\n", - "\n", - "By default, `filter_area` uses the merged mask raster and returns a new raster, unless input/output layers are specified by `layer_name` and `dest_layer_name`. \n", - "\n", - "By default, its `shape_value` parameter is 1, indicating that we are only interested in finding all groups of cells with value 1 (suitable) for elimination. We specify the threshold with the `min_area` parameter.\n", - "\n", - "Note: the `filter_area` method may take a long time (5 or more minutes depending on the complexity of your layer and your computational setup). The method relies upon `rasterio.rasterize`, see performance notes: https://rasterio.readthedocs.io/en/latest/api/rasterio.features.html#rasterio.features.rasterize\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For example, if we focus on Guangdong province in Southern China from the merged mask, we notice that there are many small islands in the ocean that are marked as suitable areas. We want to exclude these small regions from our merged mask." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(china.merged_mask.read(1)[4800:5300, 5700:6600], interpolation=\"none\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Call `filter_area` to remove all contiguous suitable region shapes smaller than 100 km$^2$:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.merged_mask = geodata.mask.filter_area(china, min_area=100)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There shapes are removed in the new merged_mask." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(china.merged_mask.read(1)[4800:5300, 5700:6600], interpolation=\"none\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extracting Shapes from Masks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Sometimes the user needs to generate masks and perform analysis for a collection of regions (e.g., at the state/province level). The purpose of shape extraction (`extract_shapes`) is to separate `merged_mask` values for each region, with the result a dictionary of name-mask pairs in the `shape_mask` attribute of the mask object. The values of `shape_mask` will be 0 outside of the shape, and will be `merged_mask` inside of the shape." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For the purpose of this demonstration, we will only select the province of Jiangsu, Zhejiang, and Shanghai." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "china_shapes_subset = china_shapes[\n", - " china_shapes[\"name\"].isin([\"Jiangsu\", \"Zhejiang\", \"Shanghai\"])\n", - "]\n", - "china_shapes_subset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Converting the filtered shape dictionary to a python dictionary as the input for `extract_shapes`, where the keys for the dictionary will be the names of the new extracted shape layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china_shapes_subset = (\n", - " china_shapes_subset[[\"name\", \"geometry\"]].set_index(\"name\")[\"geometry\"].to_dict()\n", - ")\n", - "china_shapes_subset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Extract the shapes from the merged_mask. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.extract_shapes(china_shapes_subset)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The resulting mask object contains the dictionary `shape_mask` with the extracted values:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Saving and Loading Masks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.save_mask()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With the mask saved, the user can now load the layers or shapes with `xarray` instead if preferred." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "shape_xr_lst = china.load_shape_xr()\n", - "shape_xr_lst[\"Zhejiang\"].plot()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Optional: closing all the files when saving the mask. This can avoid possible write permission error." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china.save_mask(close_files=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Loading a previously saved mask." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china_2 = geodata.mask.load_mask(\"china\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "china_2" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial: Typical Mask Creation Workflow" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Geodata is able to process geospatial data to extract cutouts over specified geographies. Built off the [rasterio library](https://rasterio.readthedocs.io/en/latest/quickstart.html), the **mask** module imports rasters and shapefiles, merges and flattens multiple layers together, and extracts subsetted cutout data from merged masks and shapefiles.\n", + "\n", + "Functionalities explored in this notebook:\n", + "\n", + "- [Creating a mask object, adding and manipulating layers](#mask-creation--adding-and-manipulating-layers)\n", + "- [Opening a shapefile and adding shape features as layers](#adding-shape-features-as-a-layer)\n", + "- [Merging and flattening layers](#merging-and-flattening-layers)\n", + "- [Eliminate small contiguous areas](#eliminate-small-contiguous-areas)\n", + "- [Extracting shapes from mask](#extracting-shapes-from-masks)\n", + "- [Saving and loading masks](#saving-and-loading-masks)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To start, import the geodata package and required libraries. We can also import the `geodata.mask.show()` method for simplicity of its use." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import geopandas as gpd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "import geodata\n", + "from geodata.mask import show" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Additionally, we use [cartopy](https://scitools.org.uk/cartopy/docs/latest/tutorials/using_the_shapereader.html#cartopy.io.shapereader.Reader) to download some common administrative region shapes, but user-provided shapefiles will also work:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import cartopy.io.shapereader as shpreader" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Shapefiles and Rasters\n", + "\n", + "We will use the following geotiff and shape files for this demo:\n", + "\n", + "\n", + "- `china_modis.tif`\n", + "\n", + " We downloaded the MODIS land cover data, which uses satellite remote sensing data to estimate the land use type on an annual basis. See: [EarthData_MCD12Q1](https://lpdaac.usgs.gov/products/mcd12q1v006/).\n", + "\n", + " We will use the IGBP classification ('LC_Type1') which has 17 different land use characterizations (the corresponding data thus takes values from 1.0 to 17.0).\n", + " All the \"Bands\" are listed here: [Google_earth_engine_MODIS_006_MCD12Q1](https://developers.google.com/earth-engine/datasets/catalog/MODIS_006_MCD12Q1#bands)\n", + "\n", + "- `china_elevation.tif` and `china_slope.tif`\n", + "\n", + " These two rasters are based on the elevation map from: [Google_earth_engine_MODIS_CGIAR_SRTM90_V4](https://developers.google.com/earth-engine/datasets/catalog/CGIAR_SRTM90_V4?hl=en). Slope was computed in degrees using the 4-connected neighbors of each pixel. \n", + "\n", + "\n", + "- `UNEP_WDPA_China` Shapefiles\n", + "\n", + " We downloaded the environmental protected area for China from: [ProtectedPlanet_China](https://www.protectedplanet.net/country/CHN). These shapefiles are distributed among 3 subfolders upon successful download and decompression due to the large size. We will create path variables for all three subfolders and we will only take the polygon shapes.\n", + "\n", + " Alternatively, We can also retrieve the environmental protected area from Google Earth Engine: [Google_earth_engine_WCMC_WDPA](https://developers.google.com/earth-engine/datasets/catalog/WCMC_WDPA_current_polygons). The shapefile will contain the protected shapes from entire world (and the size is slightly over 1 GB), and additional data cleaning will be necessary if the user wants just the shapes within China. " + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "modis_path = \"data/china_modis.tif\"\n", + "elevation_path = \"data/china_elevation.tif\"\n", + "slope_path = \"data/china_slope.tif\"\n", + "\n", + "wdpa_shape_path_0 = \"data/shapefiles/0/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"\n", + "wdpa_shape_path_1 = \"data/shapefiles/1/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"\n", + "wdpa_shape_path_2 = \"data/shapefiles/2/WDPA_WDOECM_Nov2021_Public_CHN_shp-polygons.shp\"" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us get province shapes from `cartopy` and save the path as `prov_path`. This can also be the path to user-supplied shape files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "prov_path = shpreader.natural_earth(\n", + " resolution=\"10m\", category=\"cultural\", name=\"admin_1_states_provinces\"\n", + ")\n", + "prov_path" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load the shapes contained in path `prov_path` using the `geopandas` library." + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": true + }, + "source": [ + "all_shapes = gpd.read_file(prov_path, encoding=\"utf-8\")\n", + "all_shapes.head(2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "GeoPandas data filtering with GeoDataFrame is identical to pandas. Let us select all the rows that contains shape within China." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china_shapes = all_shapes[all_shapes[\"admin\"] == \"China\"]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, to load the WDPA environmental protected shapefiles as a layer in the china mask, we will use the GeoPandas library. `gpd.read_file()` will return a GeoPandas dataframe including shape attributes and geometry given the file path. Like Pandas, we can read multiple dataframes and concat them together. In the code below, we will create one GeoPandas dataframe from three paths that we have for the Chinese environmental protected shapes." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "wdpa_shapes = pd.concat([\n", + " gpd.read_file(wdpa_shape_path_0),\n", + " gpd.read_file(wdpa_shape_path_1),\n", + " gpd.read_file(wdpa_shape_path_2)\n", + "])\n", + "wdpa_shapes.head(2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mask Creation & Adding and Manipulating Layers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The mask object consists of multiple layers and manipulations performed on them. To add a layer, the four methods below perform same functions. A user may add a layer to the mask by specifying paths when a new instance is created, or use the `add_layer` method. We will add the following two files: `china_elevation.tif`, and `china_modis.tif` to the `China` mask, and name them `elevation` and `modis` layers." + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": true + }, + "source": [ + "# Method 1: Initialize one layer, add one layer\n", + "china = geodata.Mask(\"China\", layer_path=elevation_path)\n", + "china.rename_layer(\"china_elevation\", \"elevation\")\n", + "china.add_layer(modis_path, layer_name=\"modis\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Method 2: Initialize empty, add two layers using dict\n", + "china = geodata.Mask(\"China\")\n", + "china.add_layer(layer_path={\"elevation\": elevation_path, \"modis\": modis_path})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Method 3: Initalize with two layers passed as list\n", + "china = geodata.Mask(\n", + " \"China\", layer_path=[elevation_path, modis_path], layer_name=[\"elevation\", \"modis\"]\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Method 4: Initialize with two layers passed as dict\n", + "china = geodata.Mask(\n", + " \"China\", layer_path={\"elevation\": elevation_path, \"modis\": modis_path}\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Display the mask object in the jupyter notebook:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each mask object has several attributes:\n", + "- `layers`: a dictionary of name (key) - rasterio file opener (values). The <\\open DatasetReader> can be the input for many other mask methods for the module. \n", + "- `merged_mask`: the merged and flatten mask of its layers, the merged raster from `layers`\n", + "- `shape_mask`: similar to the `layers` attribute, but a dictionary of extracted shapes from the merged mask by default. Users may also extracted shape masks from specified layers in `self.layers`.\n", + "- `saved`: whether this mask object has been saved locally.\n", + "- `mask_dir`: the directory to save the mask object, by default it should be the mask dir in config.py." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Show the `slope` layer in mask `china`. The `show` method will always try to show the proper latitude and longitude, unless we call it `show(layer, lat_lon = False)`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.layers[\"elevation\"]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "show(china.layers[\"elevation\"], title=\"Elevation of China in meters\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.layers" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Some useful methods to examine the layers**\n", + "\n", + "- `china.get_res()`: get resolution of each layer, in lat-lon coordinates\n", + "- `china.get_res(product = True)`: get grid cell size, in product of lat-lon coordinate differences\n", + "- `china.get_bounds()`: get bounds, in lat-lon coordinates" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.get_bounds()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the modis layer has a very different bounding box then the slope layer in lat-lon coordinate system. This is because the modis layer was converted to the lat-lon CRS from a different CRS when it was added to the object. The following section will explore CRS conversion." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### CRS conversion, trimming, and cropping (Optional)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Method `open_tif` can open a layer without adding it to the layer, this allows us to visualize it before-hand. It is a good practice to close the raster after opening it to avoid writing permission conflict issues. Closing the raster below does not involve any layer operation associated with the mask object. " + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "modis_opener = geodata.mask.open_tif(modis_path, show_raster=True)\n", + "modis_opener.close()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can use `remove_layer` method to remove a layer to mask `china`. This method will properly close the raster file, because the raster file would remain open after being added to the mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.remove_layer(\"modis\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `add_layer` method incorporates coordinate reference system (CRS) conversion to lat-lon (EPSG:4326), if necessary. Note that this method will overwrite the layer by default, if it is in the object already, unless the user specifies `replace=False`. \n", + "\n", + "The method will automatically trim the all-zero columns/rows. By default, the paramater `trim` is set to `True`. If we do not set it to True, we might generate a converted raster with new CRS but many all-zero columns and rows." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.add_layer(modis_path, \"modis\", trim=False)\n", + "show(china.layers[\"modis\"], title=\"China Modis CRS converted (No trimming)\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also crop a raster/layer with user-defined dimensions: method `crop_layer` can take either starting indices of top/left, ending indices of right/bottom, or coordinates values in lat/long to trim the raster.\n", + "\n", + "The difference between `crop_layer` and `trim_layer` is that `crop_layer` must take in user specified range to crop the raster, and `trim_layer` would remove the all zero rows and columns automatically for a raster. So that if the user do not know which index to start and end to remove the empty rows/columns, `trim_raster` is better.\n", + "\n", + "The method `crop_raster` (`geodata.mask.crop_raster`) is similar to `crop_layer` but can take a layer name as input, so that the user does not need to add a raster as a layer to call that method. (Similar method: `trim_layer`/`trim_raster`, `binarize_layer`/`binarize_raster`)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.crop_layer(\"modis\", bounds=(73, 17, 135, 54))\n", + "show(china.layers[\"modis\"], title=\"China Modis Layer Cropped\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This performs the same function by passing the layer to `crop_raster`:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.layers[\"modis\"] = geodata.mask.crop_raster(\n", + " china.layers[\"modis\"], (73, 17, 135, 54)\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Filter a layer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The mask module also supports filtering a layer based on list of categorical values, a minimum (lower) boundary, or maximum (upper) boundary.\n", + "\n", + "In the `filter_raster` method, a user may specify any of the `value` (the list of numberic values in the raster array to be selected), `max_bound`, and `min_bound` parameters to selected desired values. If the parameter `binarize` is False (by default), the method will return the original values of the raster that satisfy the conditions, otherwise the method will return 1 for the values that satisfy the conditions and 0 elsewhere." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Select Categorical Values from MODIS Layer\n", + "\n", + "Since the modis layer has 17 distinct values for different land use types, we want to create a layer of binary values, indicating unavailable land as 0, and available land as 1.\n", + "\n", + "We wish to create a mask where :\n", + "\n", + "- all forested areas (values 1-5) are 0 (i.e., unsuitable)\n", + "- all urban areas (13) are 0\n", + "- all others are 1\n", + "\n", + "\n", + "Let us use method `filter_raster` to create a layer of `modis_filtered` binary mask, where 1, 2, 3, 4, 5, and 13 will be unavailable land assigned 0 and the rest of the values will be 1 (available).\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "avail_values = list(set(range(1, 18)) - {1, 2, 3, 4, 5, 13})\n", + "avail_values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.layers[\"modis_filtered\"] = geodata.mask.filter_raster(\n", + " china.layers[\"modis\"], binarize=True, values=avail_values\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false + }, + "source": [ + "china.remove_layer(\"modis\")\n", + "show(china.layers[\"modis_filtered\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Filter elevation layer\n", + "\n", + "Because we cannot build renewable energy in areas with high elevation, let us set the constraint from the `elevation` layer, by using elevation < 4000m at 1 and other areas as 0. The result layer `elevation_filtered` will have only 1 and 0 as unique values." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.filter_layer(\n", + " \"elevation\", dest_layer_name=\"elevation_filtered\", max_bound=4000, binarize=True\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.remove_layer(\"elevation\")\n", + "show(china.layers[\"elevation_filtered\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Filter Slope Layer\n", + "\n", + "We also cannot build renewable energy in area with large slopes, so let us set another constraint from the `slope` layer from the slope tif file, by using slope < 20 degree at 1 and else as 0. The result layer `slope_filtered` will have only 1 and 0 as unique values." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, add the slope raster to the china mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.add_layer(slope_path, layer_name=\"slope\")\n", + "show(china.layers[\"slope\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Filter the raster, delete the old slope layer." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.filter_layer(\n", + " \"slope\", dest_layer_name=\"slope_filtered\", max_bound=20, binarize=True\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false + }, + "source": [ + "china.remove_layer(\"slope\")\n", + "show(china.layers[\"slope_filtered\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Additional Visualization Options" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can plot the provinces on a selected layer by taking `shape` input in the `show()` method. Here, we will use the `china_shapes` that we obtained from `all_shape`. Its `geometry` column is a Series of shapes (shapely.geometry or MultiPolygon) for Chinese provinces." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "show(china.layers[\"modis_filtered\"], shape=china_shapes[\"geometry\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding Shape Features as a Layer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Recall that we have previously loaded the environmental protected shapes of China in a GeoPandas dataframe." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "len(wdpa_shapes)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The three shapefiles have 78 features altogether, but we want to add all the features to one new layer instead of 78 new layers. The input shape should be a python dictionary, where there is a key for each unique shape. Also, in the `add_shape_layer` method, we will specify a `combine_name` to combine the features into one layer in this case, since we want the mask to have just one more layers, not 78 more layers.\n", + "\n", + "When adding a shapefile, we must specify the dimensions. We will also use `reference layer = 'slope_filtered'` so the new shape layer will have the same dimension with the `slope_filtered` layer. If the mask is empty and does not contain any layer, the user will have to specify the `resolution` parameter for the raster layer dimension.\n", + "\n", + "By default, this method will have paramater `exclude` that defaults to `False`. When it is true, area inside the shape is 0. When it is false, area inside the shape is 1. In this use case, however, we want 0 for area inside of the shape as they are environmental protected areas to exclude. We can just use the default method call." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.add_shape_layer(\n", + " wdpa_shapes[\"geometry\"].to_dict(),\n", + " reference_layer=\"slope_filtered\",\n", + " combine_name=\"protected\",\n", + ")\n", + "show(\n", + " china.layers[\"protected\"],\n", + " title=\"WDPA Protected area shape features as a new layer\",\n", + " grid=True,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also use the parameter `buffer` in `add_shape_layer` method to create an approximate representation of all locations within a given (perpindicular) distance of the shape object. The units for the buffer are given in kilometers.\n", + "\n", + "Note that since the units of the original shape are in lat-lon coordinates, when we add the buffer, we will need to have a CRS that has meter as unit. The program will convert the shapes to that CRS, add the buffer around shapes, then convert it back to the lat-lon CRS system. By default, we used \"EPSG:6933\", an equal area projection CRS to add buffer in kilometer. " + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "km_buffer = 20\n", + "\n", + "china.add_shape_layer(\n", + " wdpa_shapes[\"geometry\"].to_dict(),\n", + " reference_layer=\"slope_filtered\",\n", + " combine_name=\"protected_with_buffer\",\n", + " buffer=km_buffer,\n", + ")\n", + "\n", + "show(\n", + " china.layers[\"protected_with_buffer\"],\n", + " title=f\"WDPA Protected area shape with {km_buffer}km buffer\",\n", + " grid=True,\n", + ")\n", + "\n", + "china.remove_layer(\"protected_with_buffer\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Merging and Flattening Layers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to combine all layers into one, we use the `merge_layer` method which creates a new layer called `merged_mask`. This merges multiple layers together and flattens them using either **and** (default) or **sum** method, saving the result to `self.merged_mask` by default. Geospatial bounds and resolution of the output layer are in the units of the input file coordinate reference system, but by default, we will use the resolution of the layer with the best (finest) resolution for the output bounds/resolution, unless a reference layer is provided. In this case, the resolution of the merged_mask is the same with the `modis_filtered` layer. " + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.get_res()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merge_layer(attribute_save=False, show_raster=False).res" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Binary `AND` Method" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, the `merge_layer` method will use a binary 'and' method: for each grid cell, if any of the n layers are 0, then the returned `self.merged_layer` will also have 0 at that location. In other words, if all the layers indicate that a land is available (!=0), the merged result will have value 1.\n", + "\n", + "`merge_layer` may also take in an optional parameter `layers`, which is a list of layer names stored in the object, if the user does not wish to merge all layers in the object. If the user does not want to save the result to the `merged_mask` attribute, the user can specify `attribute_save = False`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# merge and plot only, do not save\n", + "china.merge_layer(attribute_save=False, layers=[\"slope_filtered\", \"modis_filtered\"])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try again with the `reference_layer` parameter:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merge_layer(\n", + " layers=[\"elevation_filtered\", \"modis_filtered\"],\n", + " reference_layer=\"elevation_filtered\",\n", + " show_raster=False,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result of the `merged_mask` method is saved to `china.merged_mask` with the same resolution as the reference layer, in this case `elevation_filtered`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merged_mask.res" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the purpose of this demonstration, we will select the `AND` method for the final merged_mask. We can also trim the border of the merged mask since the 4 layers have different boundaries. We can set the parameter `trim = True`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merge_layer(trim=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### `SUM` Method" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The sum method will add up the values from all the layers using weights. When there is no weight dict provided, all the layers for merging will have weights of 1 by default.\n", + "\n", + "Note: since we are not using the sum method to proceed to the following sections, we will keep `attribute_save = False` to prevent this method from overwriting the mask we have previously created above." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merge_layer(method=\"sum\", attribute_save=False, trim=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This distribution is completely arbitrary for the purpose of demonstration of the module: (Note: The weights do not need to have a total of 1)\n", + "- elevation_filtered: 0.15, slope_filtered: 0.1, modis_filtered: 0.3, protected: 0.45\n", + "\n", + "We will write the result to a new variable `customized_merged_layer` for continuing processing." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "customized_merged_layer = china.merge_layer(\n", + " method=\"sum\",\n", + " weights={\n", + " \"elevation_filtered\": 0.15,\n", + " \"slope_filtered\": 0.1,\n", + " \"modis_filtered\": 0.3,\n", + " \"protected\": 0.45,\n", + " },\n", + " attribute_save=False,\n", + " trim=True,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the continuous value created by `merged_mask` represents a suitability metric, we could set a minimum value of 0.8 to be considered \"suitable\" (or 1). We then apply the `filter_raster` method on the merged layer." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "customized_merged_layer = geodata.mask.filter_raster(\n", + " customized_merged_layer, min_bound=0.8, binarize=True\n", + ")\n", + "show(customized_merged_layer)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Eliminate Small Contiguous Areas" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the above methods, we might end up with many small contiguous areas that are marked suitable but surrounded by an unsuitable region. We may want to exclude such regions from renewable energy development. The `filter_area` method will remove the small contiguous suitable regions by transforming the merged mask raster to polygons/shapes, calculating the area of each polygon, and filtering out polygons that are smaller than a given threshold. Units are given in kilometer-squared (km$^2$).\n", + "\n", + "By default, `filter_area` uses the merged mask raster and returns a new raster, unless input/output layers are specified by `layer_name` and `dest_layer_name`. \n", + "\n", + "By default, its `shape_value` parameter is 1, indicating that we are only interested in finding all groups of cells with value 1 (suitable) for elimination. We specify the threshold with the `min_area` parameter.\n", + "\n", + "Note: the `filter_area` method may take a long time (5 or more minutes depending on the complexity of your layer and your computational setup). The method relies upon `rasterio.rasterize`, see performance notes: https://rasterio.readthedocs.io/en/latest/api/rasterio.features.html#rasterio.features.rasterize\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For example, if we focus on Guangdong province in Southern China from the merged mask, we notice that there are many small islands in the ocean that are marked as suitable areas. We want to exclude these small regions from our merged mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "plt.imshow(china.merged_mask.read(1)[4800:5300, 5700:6600], interpolation=\"none\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Call `filter_area` to remove all contiguous suitable region shapes smaller than 100 km$^2$:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.merged_mask = geodata.mask.filter_area(china, min_area=100)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There shapes are removed in the new merged_mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "plt.imshow(china.merged_mask.read(1)[4800:5300, 5700:6600], interpolation=\"none\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Extracting Shapes from Masks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sometimes the user needs to generate masks and perform analysis for a collection of regions (e.g., at the state/province level). The purpose of shape extraction (`extract_shapes`) is to separate `merged_mask` values for each region, with the result a dictionary of name-mask pairs in the `shape_mask` attribute of the mask object. The values of `shape_mask` will be 0 outside of the shape, and will be `merged_mask` inside of the shape." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the purpose of this demonstration, we will only select the province of Jiangsu, Zhejiang, and Shanghai." + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false + }, + "source": [ + "china_shapes_subset = china_shapes[\n", + " china_shapes[\"name\"].isin([\"Jiangsu\", \"Zhejiang\", \"Shanghai\"])\n", + "]\n", + "china_shapes_subset" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Converting the filtered shape dictionary to a python dictionary as the input for `extract_shapes`, where the keys for the dictionary will be the names of the new extracted shape layers." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china_shapes_subset = (\n", + " china_shapes_subset[[\"name\", \"geometry\"]].set_index(\"name\")[\"geometry\"].to_dict()\n", + ")\n", + "china_shapes_subset" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Extract the shapes from the merged_mask. " + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.extract_shapes(china_shapes_subset)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting mask object contains the dictionary `shape_mask` with the extracted values:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Saving and Loading Masks" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.save_mask()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the mask saved, the user can now load the layers or shapes with `xarray` instead if preferred." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "shape_xr_lst = china.load_shape_xr()\n", + "shape_xr_lst[\"Zhejiang\"].plot()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Optional: closing all the files when saving the mask. This can avoid possible write permission error." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china.save_mask(close_files=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Loading a previously saved mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china_2 = geodata.mask.load_mask(\"china\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "china_2" + ], + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/docs/source/mask/mask_troubleshoot.md b/docs/source/mask/mask_troubleshoot.md index a08ad7d6..bc8bd6c4 100644 --- a/docs/source/mask/mask_troubleshoot.md +++ b/docs/source/mask/mask_troubleshoot.md @@ -2,6 +2,13 @@ This is a document that includes possible errors for the mask module and troubleshooting information. +## `merge_layer` / `RasterioIOError` (No such file or directory) + +This was caused by in-memory (`/vsimem`) layers whose backing `MemoryFile` was closed too early. +Current geodata pins memory files for the lifetime of each layer reader; filter → merge should work +without pre-saving layers. If the error persists, see **[merge_layer_known_issues.md](merge_layer_known_issues.md)** +for historical context and workarounds for older versions. + ## No Affine Transformation If you run into this error when loading any tif file with the mask module: diff --git a/docs/source/mask/merge_layer_known_issues.md b/docs/source/mask/merge_layer_known_issues.md new file mode 100644 index 00000000..f5502895 --- /dev/null +++ b/docs/source/mask/merge_layer_known_issues.md @@ -0,0 +1,34 @@ +# `merge_layer` known issues (historical) + +```{note} +**Historical context.** Older geodata versions could raise ``RasterioIOError: No such +file or directory`` when merging **in-memory** (``/vsimem``) mask layers. Current code +pins memory files for the lifetime of each layer reader so ``filter_layer`` → +``merge_layer`` normally works without pre-saving layers to disk. +``` + +## Symptom + +``merge_layer`` (or ``merge_and`` / ``merge_sum`` after filters) fails with an error +referring to a missing path under ``/vsimem/``. + +## Cause (legacy behavior) + +Raster layers stored in GDAL memory files were sometimes closed before merge read them +back, so the virtual path was no longer valid. + +## Current behavior + +The mask module keeps layer readers alive while a ``Mask`` object uses in-memory +layers. If you still see this error on an old install, upgrade geodata or save +intermediate layers to disk before merging. + +## Workaround (older versions) + +1. Save filtered layers to GeoTIFF before ``merge_layer``. +2. Call ``save_mask(close_files=True)`` when finished, and avoid two ``Mask`` objects + opening the same files simultaneously (see [mask troubleshooting](mask_troubleshoot.md)). + +## Tests + +Regression coverage lives under ``tests/pr/mask/test_mask_merge_inmemory.py``. diff --git a/docs/source/mask/xarray_mask_tutorial.ipynb b/docs/source/mask/xarray_mask_tutorial.ipynb new file mode 100644 index 00000000..fdc86f89 --- /dev/null +++ b/docs/source/mask/xarray_mask_tutorial.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial: Applying Saved Masks with `XarrayMask`\n", + "\n", + "This notebook shows how to apply a **saved** geographic mask to model or analysis\n", + "output represented as an `xarray.Dataset` or `xarray.DataArray` — without using\n", + "`Cutout.add_mask` or `Cutout.mask`.\n", + "\n", + "For contributor notes on the xarray masking design, see\n", + "[development/xarray_mask_workflow](../development/xarray_mask_workflow.rst).\n", + "To build masks from rasters and shapefiles, see\n", + "[mask creation workflow](mask_creation_workflow.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "| Step | API | Module |\n", + "|------|-----|--------|\n", + "| Create and save a mask | `geodata.Mask` | `src/geodata/mask.py` |\n", + "| Run a model (wind, pvlib, …) | model `estimate()` | `src/geodata/model/` |\n", + "| Align mask to your grid, attach or apply | `geodata.XarrayMask` | `src/geodata/mask/xarray_mask.py` |\n", + "\n", + "**`XarrayMask` does not replace mask creation.** It loads a saved mask and applies it\n", + "to xarray data on your target grid." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "This tutorial runs **offline** using a small synthetic grid and a temporary mask\n", + "directory. The same API calls work for production masks saved under `GEODATA_ROOT`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import rasterio as ras\n", + "import shapely.geometry\n", + "import xarray as xr\n", + "from rasterio.transform import from_bounds\n", + "\n", + "from geodata import Mask, XarrayMask" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Stand in for model output\n", + "\n", + "Your analysis dataset can use `x`/`y` or `lat`/`lon`. `XarrayMask` normalizes\n", + "coordinates via `ds_reformat_index` before alignment.\n", + "\n", + "Below we use a small `(time, y, x)` dataset as if it came from a wind or PV model." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "y = np.array([30.75, 30.5, 30.25, 30.0])\n", + "x = np.array([100.0, 100.25, 100.5, 100.75])\n", + "time = np.array([\"2016-01-01T00:00:00\", \"2016-01-01T01:00:00\"], dtype=\"datetime64[ns]\")\n", + "\n", + "values = np.arange(len(time) * len(y) * len(x), dtype=np.float32).reshape(\n", + " len(time), len(y), len(x)\n", + ")\n", + "model_ds = xr.Dataset(\n", + " {\"signal\": ((\"time\", \"y\", \"x\"), values)},\n", + " coords={\"time\": time, \"y\": y, \"x\": x},\n", + ")\n", + "model_ds" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Create and save a mask (offline example)\n", + "\n", + "In practice you build masks with `Mask.add_layer`, `filter_layer`, `merge_layer`,\n", + "and `save_mask()` — see [mask creation workflow](mask_creation_workflow.ipynb).\n", + "\n", + "Mask rasters are often stored at **higher resolution** than model output.\n", + "`XarrayMask` coarsens them onto `grid` automatically.\n", + "\n", + "The helper below mirrors `tests/pr/mask/test_xarray_mask.py`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mask_dir = Path(tempfile.mkdtemp(prefix=\"geodata_xmask_tutorial_\"))\n", + "mask_name = \"tutorial_mask\"\n", + "\n", + "lon_step = float(np.abs(x[1] - x[0]))\n", + "lat_step = float(np.abs(y[1] - y[0]))\n", + "west = float(x.min() - lon_step / 2)\n", + "east = float(x.max() + lon_step / 2)\n", + "south = float(y.min() - lat_step / 2)\n", + "north = float(y.max() + lat_step / 2)\n", + "\n", + "nlon_hi = len(x) * 2\n", + "nlat_hi = len(y) * 2\n", + "transform = from_bounds(west, south, east, north, nlon_hi, nlat_hi)\n", + "\n", + "arr = np.zeros((nlat_hi, nlon_hi), dtype=np.uint8)\n", + "arr[nlat_hi // 4 : 3 * nlat_hi // 4, nlon_hi // 4 : 3 * nlat_hi // 4] = 1\n", + "\n", + "layer_path = mask_dir / \"source.tif\"\n", + "with ras.open(\n", + " str(layer_path),\n", + " \"w\",\n", + " driver=\"GTiff\",\n", + " height=arr.shape[0],\n", + " width=arr.shape[1],\n", + " count=1,\n", + " dtype=arr.dtype,\n", + " compress=\"lzw\",\n", + " crs=\"+proj=latlong\",\n", + " transform=transform,\n", + ") as dst:\n", + " dst.write(arr, 1)\n", + "\n", + "mask = Mask(name=mask_name, mask_dir=str(mask_dir))\n", + "mask.add_layer(str(layer_path), layer_name=\"source\")\n", + "mask.merge_layer(show_raster=False)\n", + "\n", + "region = shapely.geometry.box(west, south, (west + east) / 2, (south + north) / 2)\n", + "mask.extract_shapes({\"region_a\": region}, show_raster=False)\n", + "mask.save_mask()\n", + "\n", + "print(f\"Saved mask '{mask_name}' under {mask_dir}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Load and align — `XarrayMask.from_name`\n", + "\n", + "Pass your model grid so the saved mask is coarsened and aligned to the same\n", + "`x`/`y` (or `lat`/`lon`) coordinates." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "xmask = XarrayMask.from_name(mask_name, grid=model_ds, mask_dir=str(mask_dir))\n", + "xmask" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also build from an in-memory `Mask` object:\n", + "\n", + "```python\n", + "loaded = Mask.from_name(mask_name, mask_dir=str(mask_dir))\n", + "xmask = XarrayMask.from_mask(loaded, grid=model_ds)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Attach — legacy-compatible output\n", + "\n", + "`attach()` returns a dict of datasets (keys: `merged_mask`, plus any shape masks).\n", + "Each dataset contains your original variables plus `mask` and optional `area` — the\n", + "same structure as `Cutout.mask()`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "attached = xmask.attach(model_ds, include_area=True)\n", + "list(attached.keys())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "merged = attached[\"merged_mask\"]\n", + "merged" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Apply — filtered outputs\n", + "\n", + "- `mode=\"where\"` — set values outside the mask to NaN\n", + "- `mode=\"multiply\"` — set values outside the mask to zero" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "where_out = xmask.apply(model_ds, mode=\"where\", include_area=True)[\"merged_mask\"]\n", + "multiply_out = xmask.apply(model_ds, mode=\"multiply\", include_area=False)[\"merged_mask\"]\n", + "\n", + "where_out[\"signal\"].isel(time=0)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Area-weighted aggregation\n", + "\n", + "With `attach(..., include_area=True)` you can compute mask- and area-weighted\n", + "statistics over time — the same pattern as the legacy Cutout workflow." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "ds = attached[\"merged_mask\"]\n", + "weighted_mean = (\n", + " (ds[\"signal\"] * ds[\"mask\"] * ds[\"area\"]).sum(dim=[\"lat\", \"lon\"])\n", + " / (ds[\"mask\"] * ds[\"area\"]).sum(dim=[\"lat\", \"lon\"])\n", + ")\n", + "weighted_mean" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Production usage\n", + "\n", + "When your mask is already saved under the default mask directory (`GEODATA_ROOT`):\n", + "\n", + "```python\n", + "xmask = XarrayMask.from_name(\"china\", grid=output_ds) # uses geodata.config.MASK_DIR\n", + "masked = xmask.apply(output_ds, mode=\"where\")\n", + "```\n", + "\n", + "### Typical pipeline\n", + "\n", + "1. `output_ds = model.estimate(...)`\n", + "2. `xmask = XarrayMask.from_name(\"my_mask\", grid=output_ds, mask_dir=...)`\n", + "3. `xmask.attach(output_ds)` or `xmask.apply(output_ds, ...)`\n", + "\n", + "### See also\n", + "\n", + "| Topic | Page |\n", + "|-------|------|\n", + "| Create masks from GIS layers | [mask_creation_workflow](mask_creation_workflow.ipynb) |\n", + "| Legacy Cutout masking | [mask_on_cutout](../legacy/mask_on_cutout.ipynb) |\n", + "| Xarray masking design notes (contributors) | [xarray_mask_workflow](../development/xarray_mask_workflow.rst) |\n", + "| Migration plan (contributors) | [mask_xarray_migration_plan](../development/mask_xarray_migration_plan.md) |\n", + "| Automated examples | `tests/pr/mask/test_xarray_mask.py`, `tests/pr/test_wind_xarraymask_integration.py` |" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/docs/source/modeling/era5_outputs.md b/docs/source/modeling/era5_outputs.md new file mode 100644 index 00000000..834042c2 --- /dev/null +++ b/docs/source/modeling/era5_outputs.md @@ -0,0 +1,58 @@ +# ERA5 model outputs + +After you download ERA5 data (see :ref:`downloading-era5-data` in the +[Dataset module overview](../datasets/overview.rst)), **models** turn raw files into +analysis-ready time series. Masking applies afterward on model results (see +[Mask tutorials](../mask/xarray_mask_tutorial.ipynb)). + +This page is a short catalog of common **model** outputs on the current tested path. It +does not describe legacy ``Cutout`` / ``geodata.convert`` products (see +[Legacy MERRA2 outputs](../legacy/merra2/merra2_outputs.md)). + +## Wind generation time-series + +Hub-height **capacity factor** (``cf``) from ERA5 3D wind: + +| Step | Component | +|------|-----------| +| Dataset | ``wind_3d_hourly`` (or ``wind_3d_hourly_test`` for offline fixtures) | +| Model | ``WindInterpolationModel`` — [Wind modeling](wind/index.rst), [interpolation tutorial](wind/interpolation.rst) | + +```python +from geodata.datasets import load_dataset +from geodata.model.wind import WindInterpolationModel + +ds = load_dataset("wind_3d_hourly")(years=slice(2016, 2016), months=slice(1, 1)) +if not ds.downloaded: + ds.download() + +model = WindInterpolationModel(ds) +model.prepare() +cf = model.estimate(turbine="Vestas_V112_3MW", years=slice(2016, 2016), months=slice(1, 1)) +``` + +## Wind speed time-series + +Same dataset and model; pass ``height=`` instead of ``turbine=``: + +```python +wind_speed = model.estimate(height=100.0, years=slice(2016, 2016), months=slice(1, 1)) +``` + +## Solar photovoltaic generation time-series + +Hourly **AC power** (``ac``) and **capacity factor** (``pv``): + +| Step | Component | +|------|-----------| +| Dataset | ``wind_solar_hourly`` (or ``wind_solar_hourly_test`` for offline fixtures) | +| Model | ``Pvlib`` — [PVLib modeling](pvlib/index.rst) | + +After ``init_pv_system()`` and ``init_model_config()``, call ``estimate()`` (see the +pvlib docs for ``compact_output`` and spatial subsetting). + +## See also + +- [ERA5 CDS setup](../datasets/era5.rst) +- [Offline ERA5 fixtures](../development/offline-era5-fixture-datasets.md) +- [Supported input/output formats](../quick_start/input_output.md) diff --git a/docs/source/modeling/pvlib/index.rst b/docs/source/modeling/pvlib/index.rst new file mode 100644 index 00000000..68f4a41d --- /dev/null +++ b/docs/source/modeling/pvlib/index.rst @@ -0,0 +1,174 @@ +PVLib Modeling +============== + +PVLib is a Python library for modeling solar photovoltaic systems. It provides a set of tools for modeling the performance of solar photovoltaic systems + +How to use the model +--------------------- + +The PVLib models are imported from the `pvlib` module. + +Step 1: Import the necessary libraries +---------------------------------------- + +To get started, we need to import the required libraries. We will import +the `pvlib` from the `geodata` library, as well as any other +libraries needed for data handling and visualization. + +.. code:: Python + + import xarray as xr + + from geodata.datasets import load_dataset + from geodata.model.pvlib import Pvlib + +Step 2: Load the dataset +------------------------ + +Next, we need to load the dataset that contains the solar irradiance data. +We will use the `wind_solar_hourly` dataset from the ERA5 dataset. + +.. code:: Python + + # Load the dataset + ds_cls = load_dataset("wind_solar_hourly") + ds = ds_cls( + years = slice(2016, 2016), + months = slice(1, 1) + ) + if not ds.downloaded: + ds.download() # Download the data if we don't have it locally + print(ds.downloaded) # Check if the dataset is downloaded. Should return True. + +Step 3: Create the model with specific configs +---------------------------------------------- + +Next, we need to create the model with specific configs. + +.. code:: Python + + model = Pvlib(ds) + +Two configurations are required: (1) **PV system setup** — physical array geometry (tilt, azimuth), module and inverter from the SAM database, and racking; (2) **Model config** — algorithms for clearsky irradiance, transposition, solar position, airmass, DC/AC conversion (CEC, Sandia), and losses (AOI, spectral, ohmic). +Following is an example of how to create the model with specific configs. + +.. code:: Python + + # create the pv_system + n_mods = 50 + n_strings = 1 + cec_modules = model.retrieve_sam('CECMod') + module = cec_modules['Kaneka_U_SA105'] + inv = model.retrieve_sam("CECInverter")['Fronius_USA__CL_33_3_Delta__208V_'] + model.init_pv_system( + arrays = None, + surface_tilt=35, + surface_azimuth=180, + racking_model = 'open_rack', + module_parameters=module, + modules_per_string = n_mods, + module_type = 'glass_polymer', + module = 'Kaneka_U_SA105', + strings_per_inverter = n_strings, + inverter_parameters=inv + ) + +.. code:: Python + + model.init_model_config( + clearsky_model= 'haurwitz', + transposition_model='perez', + solar_position_method= 'nrel_numpy', + airmass_model= 'kastenyoung1989', + dc_model='cec', + ac_model='sandia', + aoi_model="physical", + spectral_model='first_solar', + dc_ohmic_model='no_loss' + ) + +Step 4: Estimate the capacity factor +------------------------------------ + +Next, we can estimate the AC Power and PV capacity using the model. + +.. code:: Python + + cf = model.estimate( + years = slice(2016, 2016), + months = slice(1, 1), + xs = slice(8, 10), # Optional: longitude subset + ys = slice(48, 46), # Optional: latitude subset (see below) + ) + print(cf) + +The output will be an xarray Dataset containing the estimated AC power (``ac``) and +capacity factor (``pv``) for the specified region and time period. + +Estimate options +---------------- + +All models inherit a common pattern for **time** and **space** subsetting via +``estimate()``. The PVLib model adds one extra output option. + +Temporal subsetting +~~~~~~~~~~~~~~~~~~~ + +Pass ``years`` and ``months`` as ``slice`` objects to limit the period processed. +Omit either argument to use the prepared model's full range (subject to what was +available when ``prepare()`` ran). + +Spatial subsetting (``xs``, ``ys``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pass ``xs`` and ``ys`` as ``slice(start, stop)`` to restrict longitude (``x``) and +latitude (``y``). Omit either argument to keep the full horizontal extent of the +prepared dataset. + +Geodata **normalizes slice bounds** before calling xarray's ``.sel()``. You can pass +bounds in either order (for example ``ys=slice(48, 46)`` for a band in central +Europe) and still get a non-empty selection. This matters for ERA5-style grids where +latitude is often stored in **descending** order: a naive ``slice(46, 48)`` would +return no points without normalization. + +.. code:: Python + + # Equivalent selections on a descending-latitude grid: + cf_a = model.estimate(years=slice(2016, 2016), months=slice(1, 1), ys=slice(48, 46)) + cf_b = model.estimate(years=slice(2016, 2016), months=slice(1, 1), ys=slice(46, 48)) + +Compact output (``compact_output``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``estimate()`` returns a compact dataset with only two data variables: + +- ``ac`` — AC power (W) +- ``pv`` — capacity factor (AC output normalized by module nameplate) + +Set ``compact_output=False`` to retain **all intermediate weather and ModelChain +columns** per grid cell (irradiance components, temperature, wind, and other inputs +used along the chain). Use this for debugging or when you need columns beyond +``ac`` and ``pv``; the result is larger and slower to write. + +.. code:: Python + + # Default: only ac and pv + cf = model.estimate( + years=slice(2016, 2016), + months=slice(1, 1), + compact_output=True, + ) + list(cf.data_vars) # ['ac', 'pv'] + + # Full per-coordinate table (debugging / downstream analysis) + full = model.estimate( + years=slice(2016, 2016), + months=slice(1, 1), + compact_output=False, + ) + list(full.data_vars) # ac, pv, plus weather and intermediate columns + +.. toctree:: + :maxdepth: 1 + :caption: Tutorials on Specific Models + diff --git a/docs/source/modeling/wind/extrapolation.rst b/docs/source/modeling/wind/extrapolation.rst deleted file mode 100644 index a983806e..00000000 --- a/docs/source/modeling/wind/extrapolation.rst +++ /dev/null @@ -1,166 +0,0 @@ -Tutorial: Estimate Wind Speed with Extrapolation -================================================ - -In this tutorial, we will learn how to estimate wind speed using the extrapolation model - from the geodata library. - -.. warning:: - Performing wind speed estimation using extrapolation requires a dataset with known - wind speed values at **multiple** locations. - - Currently, only the :code:`weather_data_config` :code:`slv_flux_hourly` from the MERRA2 dataset - contains the necessary wind speed data for extrapolation. - - Therefore, all of the information below only applies with :code:`slv_flux_hourly` or cutouts - derived from it. Using any other dataset will lead to a :code:`ValueError`. - -Step 1: Import the necessary libraries ----------------------------------------- - -To get started, we need to import the required libraries. We will import the `WindExtrapolationModel` from the `geodata` library, as well as any other libraries needed for data handling and visualization. - -.. code:: Python - - import xarray as xr - - from geodata.datasets import load_dataset - from geodata.model.wind import WindExtrapolationModel - - -Step 2: Load the dataset ------------------------- - -Next, we need to load the dataset that contains the wind speed data. We will use the `slv_flux_hourly` dataset from the ERA5 dataset. - -.. code:: Python - - # Load the dataset - ds_cls = load_dataset("slv_flux_hourly") - ds = ds_cls( - years=slice(2006, 2006), - months=slice(1, 1), - bounds=[-10, 35, 10, 45] # Optional: specify the bounding box - ) - - if not ds.downloaded: - ds.download() # Download the data if we don't have it locally - - print(ds.downloaded) # Check if the dataset is downloaded. Should return True. - - -Step 3: Compute extrapolation parameters --------------------------------------------- -The extrapolation is separated into two steps, first estimating extrapolation parameters -using linear regression, and second extrapolating to desired heights. -First, we compute the extrapolation parameters. -For more information on the model, see the section below: `How the Extrapolation Model Works`_. - -.. code:: Python - - # Create a model based on the above dataset. The model will be associated with - # the dataset forever. If you wish to use a different dataset, you will need to - # create a new model. - - model = WindExtrapolationModel(ds) - model.prepare() - -If you have already prepared a cutout with the config :code:`slv_flux_hourly`, you -can also pass -that into the model as well. The model treats dataset and cutouts indifferently. -Simply replace :code:`ds` with your cutout variable. - -.. note:: - The `prepare` method computes the necessary parameters for the extrapolation model - based on the loaded dataset. Everything will be saved under the :code:`models` - directory under the path :code:`GEODATA_ROOT`. - -.. note:: - It is not necessary to call the `prepare` method every time you want to perform - extrapolation. You only need to call it once after loading the dataset. From that - point on, you can load and use the model directly without re-preparing it. - -Step 4: Estimate using the extrapolation model ----------------------------------------------- - -Now that we have prepared the model, we can perform the extrapolation to estimate wind -speed at the desired locations. Suppose we want to estimate the wind speed at a height -of 60 above ground during January of 2006 for the entire region covered by the original -dataset, we can do this as follows: - -.. code:: Python - - estimated_wind_speed = model.estimate( - height=60, - years=slice(2006, 2006), - months=slice(1, 1), - ) - -This will return an xarray DataArray containing the estimated wind speed values. Note -that you can also select a subset area by passing in :code:`xs=slice(start, end)` -and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. - - -Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model --------------------------------------------------------------------------------- - -Geodata also supports a limited set of wind turbine models to estimate the capacity -factor (CF) of a wind turbine directly. To get a list of available wind turbine models, -you can use the `get_available_windturbines` function: - -.. code:: Python - - from geodata.resource import get_available_windturbines - - turbines = get_available_windturbines() - print(turbines) # List of available wind turbine configurations - - -To estimate the capacity factor of a wind turbine, you can use the `estimate` method -and passign in the `turbine` parameter with the name of the wind turbine model. - -.. code:: Python - - # Estimate the capacity factor for a specific wind turbine model - estimated_cf: xr.Dataset = model.estimate( - turbine="Vestas_V112_3MW", # Example wind turbine model - years=slice(2006, 2006), - months=slice(1, 1), - ) - - print(estimated_cf) # Display the estimated capacity factor - - -The output will be an xarray Dataset containing the estimated capacity factor values -for the specified wind turbine model over the given time period and region. - - -How the Extrapolation Model Works ---------------------------------- - -The model calculates hub height wind speed from MERRA2, extrapolating the variables in -MERRA's tavg1_2d_slv_Nx data collection, which is a set of the time-averaged -single-layer diagnostics. - -Specifically, the variables we use for extrapolation are: 2-m wind (U2M, V2M, in m/s), -10-m wind (U10M, V10M), 50-m wind (U50M, V50M), and the zero-plane displacement -height (DISPH, in meters). Additionally, we also use the wind speed at MERRA2's lowest -model level (ULML, VLML, in m/s), the height of the lowest model level -(HLML, in meters), may vary depending on the location. We can obtain the wind speed at -any given location and height by computing the norm of the vector sum of the U and V -components. - - -The hub height wind speed can be calculated as - -.. math:: - \nu = \alpha \ln\left(\frac{H - d}{z}\right) - -.. math:: - z = e^{-\beta/\alpha} - -where :math:`\nu` is the hub height wind speed, :math:`\alpha` is the best-fit slope -from a linear regression of wind speeds on vertical heights, :math:`\ln` is the natural logarithm, :math:`H` is the hub height, -:math:`d` is the zero-plane displacement height, and :math:`\beta` is the intercept -from the linear regression fit. - -Here we estimate :math:`\alpha` and :math:`\beta` fitting a simple linear regression model to the heights and wind speeds in the data. diff --git a/docs/source/modeling/wind/index.rst b/docs/source/modeling/wind/index.rst index e6a93019..4ff14d70 100644 --- a/docs/source/modeling/wind/index.rst +++ b/docs/source/modeling/wind/index.rst @@ -1,9 +1,9 @@ Wind Modeling ============= -Starting from geodata v0.2.0, geodata's capability to model and estimate wind speed have -been from the cutout module to a separate wind module. This module has the capability to -estimate wind speed with two modes: interpolation and extrapolation. +Starting from geodata v0.2.0, geodata's wind modeling capability lives in a separate +``geodata.model.wind`` module. The **supported ERA5 path** uses vertical spline +**interpolation** on ``wind_3d_hourly`` data (see :doc:`interpolation`). How to use the models --------------------- @@ -59,7 +59,33 @@ different dataset, you will need to create a new model. model = WindInterpolationModel(ds) model.prepare() # Prepare the model - print(model.prepared) # Check if the model is prepared. Should return True. + print(model.prepared) # Check if the model is prepared. Should return True. + + +Preparing the model (``prepare``, ``prepared``, ``force``) +---------------------------------------------------------- + +Wind models must be **prepared** before ``estimate()``. Preparation reads the +downloaded ERA5 files, computes month-by-month coefficients (B-spline parameters for +interpolation), and writes cached results under ``GEODATA_ROOT/models/`` (see +:doc:`/quick_start/packagesetup`). + +- ``model.prepared`` — ``True`` when every month in the model's year/month range has + cached outputs on disk. +- ``model.prepare()`` — run once after ``ds.downloaded`` is ``True``. Safe to skip if + already prepared. +- ``model.prepare(force=True)`` — delete and recompute cached months (use after changing + ``years`` / ``months`` / ``bounds`` on the source dataset, or when upgrading geodata). + +``estimate()`` raises if the model is not prepared. Pvlib does **not** use this +prepare step; only wind models do. + +.. code:: Python + + if not model.prepared: + model.prepare() + # After changing the source time range or domain: + # model.prepare(force=True) Once the model is prepared, we can use it to estimate wind speed at desired heights. @@ -76,9 +102,58 @@ Once the model is prepared, we can use it to estimate wind speed at desired heig The above demonstrates the typical workflow. More model-specific details can be found in each model's respective tutorial as well as in the API reference. +Estimate options +---------------- + +Wind models share the same ``estimate()`` subsetting interface (defined on +``BaseModel`` in ``geodata.model``). + +Temporal subsetting +~~~~~~~~~~~~~~~~~~~ + +Use ``years`` and ``months`` slices to limit the estimation period. For example, +``years=slice(2006, 2006), months=slice(1, 1)`` processes January 2006 only. + +Spatial subsetting (``xs``, ``ys``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pass ``xs`` and ``ys`` as ``slice(start, stop)`` to restrict longitude (``x``) and +latitude (``y``). Omit either argument to use the full horizontal domain of the +prepared source. + +Geodata **normalizes slice bounds** before ``xarray.Dataset.sel()``. You may pass +``slice(high, low)`` or ``slice(low, high)``; the helper resolves the inclusive +range and matches the coordinate's ascending or descending order (ERA5 latitude is +typically descending). Without this, a slice like ``ys=slice(46, 48)`` on a +descending ``y`` axis can incorrectly return an empty selection. + +.. code:: Python + + # Subregion over central Europe — bounds order does not matter + wind_speed = model.estimate( + height=100.0, + years=slice(2006, 2006), + months=slice(1, 1), + xs=slice(8, 10), + ys=slice(48, 46), + ) + +Wind-specific arguments +~~~~~~~~~~~~~~~~~~~~~~~ + +Pass **either**: + +- ``height=`` — hub-height or AGL wind speed (``WindInterpolationModel`` on + ``wind_3d_hourly``), or +- ``turbine=""`` — capacity factor (``cf``) from a turbine YAML under + ``geodata.resources.windturbine``. The name is the YAML stem (e.g. + ``Vestas_V112_3MW``). See :doc:`interpolation` Step 5 for usage and what + ``cf`` represents. + +List available turbines with ``geodata.resource.get_available_windturbines()``. + .. toctree:: :maxdepth: 1 :caption: Tutorials on Specific Models interpolation - extrapolation diff --git a/docs/source/modeling/wind/interpolation.rst b/docs/source/modeling/wind/interpolation.rst index 788bba56..04751db6 100644 --- a/docs/source/modeling/wind/interpolation.rst +++ b/docs/source/modeling/wind/interpolation.rst @@ -127,16 +127,27 @@ by the original dataset, we can do this as follows: ) -This will return an xarray Dataset containing the estimated wind speed values. Note -that you can also select a subset area by passing in :code:`xs=slice(start, end)` -and/or :code:`ys=slice(start, end)` parameters to the `estimate` method. +This will return an xarray Dataset containing the estimated wind speed values. You can +also restrict the horizontal domain with ``xs`` and ``ys`` (see +:doc:`/modeling/wind/index` — **Estimate options** for slice-order behavior on +ERA5 grids). + +.. code:: Python + + estimated_wind_speed = model.estimate( + height=60.0, + years=slice(2006, 2006), + months=slice(1, 1), + xs=slice(8, 10), + ys=slice(48, 46), + ) Step 5: Estimate Wind Turbine Capacity Factor (CF) using the interpolation model -------------------------------------------------------------------------------- Geodata also supports a limited set of wind turbine models to estimate the capacity factor (CF) of a wind turbine directly. To get a list of available wind turbine models, -you can use the `get_available_windturbines` function: +you can use the ``get_available_windturbines`` function: .. code:: Python @@ -146,20 +157,39 @@ you can use the `get_available_windturbines` function: print(turbines) # List of available wind turbine configurations -To estimate the capacity factor of a wind turbine, you can use the `estimate` method -and passign in the `turbine` parameter with the name of the wind turbine model. +Pass the YAML **stem** (filename without ``.yaml``) as ``turbine`` — for example +``Vestas_V112_3MW`` for ``src/geodata/resources/windturbine/Vestas_V112_3MW.yaml``. .. code:: Python - # Estimate the capacity factor for a specific wind turbine model - estimated_cf: xr.Dataset = model.estimate( - turbine="Vestas_V112_3MW", # Example wind turbine model + estimated_cf = model.estimate( + turbine="Vestas_V112_3MW", years=slice(2006, 2006), months=slice(1, 1), ) - print(estimated_cf) # Display the estimated capacity factor + print(estimated_cf) + +Understanding the output +~~~~~~~~~~~~~~~~~~~~~~ + +``estimate(turbine=...)`` returns an ``xarray.DataArray`` named ``cf`` with dimensions +``(time, x, y)`` when those coordinates are present. + +Geodata computes CF in three steps: + +1. **Hub-height wind speed** — interpolate to the turbine's ``HUB_HEIGHT`` from the + YAML (same vertical spline as Step 4, but at the turbine height rather than a + height you pass manually). +2. **Power from the power curve** — map wind speed to power (MW) by interpolating the + tabulated ``V`` / ``POW`` pairs in the turbine YAML. +3. **Normalize** — ``cf = power / P``, where ``P`` is the rated power (maximum value + in ``POW``). +So ``cf`` is a **dimensionless capacity factor** in ``[0, 1]`` (values can exceed 1 +briefly if the curve extrapolates above rated power). Values outside the tabulated +wind-speed range use SciPy's ``interp1d`` extrapolation — treat edge cases with care +in sensitivity analysis. -The output will be an xarray Dataset containing the estimated capacity factor values -for the specified wind turbine model over the given time period and region. +For implementation details, see ``WindBaseModel._estimate_power`` in the +:ref:`API reference `. diff --git a/docs/source/quick_start/input_output.md b/docs/source/quick_start/input_output.md index e08ddea0..03be7cb7 100644 --- a/docs/source/quick_start/input_output.md +++ b/docs/source/quick_start/input_output.md @@ -10,6 +10,9 @@ ### MERRA2 +MERRA-2 is supported through the **legacy** ``Dataset`` / ``Cutout`` API only. See +[Legacy workflow → MERRA2](../legacy/merra2/index.md) for download and cutout tutorials. + * [MERRA2 hourly, single-level surface flux diagnostics](https://disc.gsfc.nasa.gov/datasets/M2T1NXFLX_5.12.4/summary) * [MERRA2 monthly mean, single-level surface flux diagnostics](https://disc.gsfc.nasa.gov/datasets/M2TMNXFLX_5.12.4/summary) * [MERRA2 daily mean, single-level diagnostics](https://disc.gsfc.nasa.gov/datasets/M2SDNXSLV_5.12.4/summary) @@ -31,15 +34,15 @@ The following outputs are currently supported for climate data: **Wind** -* Wind generation time-series ([MERRA2](../datasets/merra2/merra2_outputs.md#wind-generation-time-series), [ERA5](../datasets/era5/era5_outputs.md#wind-generation-time-series)) -* Wind speed time-series ([MERRA2](../datasets/merra2/merra2_outputs.md#wind-speed-time-series), [ERA5](../datasets/era5/era5_outputs.md#wind-speed-time-series)) -* Wind power density time-series ([MERRA2 only](../datasets/merra2/merra2_outputs.md#wind-power-density-time-series)) +* Wind generation time-series ([ERA5 model outputs](../modeling/era5_outputs.md#wind-generation-time-series), [wind modeling](../modeling/wind/index.rst), [ERA5 setup](../datasets/era5.rst)) +* Wind speed time-series ([ERA5 model outputs](../modeling/era5_outputs.md#wind-speed-time-series), [wind modeling](../modeling/wind/index.rst), [ERA5 setup](../datasets/era5.rst)) +* Wind power density time-series ([MERRA2 only (legacy)](../legacy/merra2/merra2_outputs.md#wind-power-density-time-series)) **Solar** -* Solar photovoltaic generation time-series ([ERA5 only](../datasets/era5/era5_outputs.md#solar-photovoltaic-generation-time-series)) -* PV generation time-series ([MERRA2 only](../datasets/merra2/merra2_outputs.md#pv-generation-time-series)) +* Solar photovoltaic generation time-series ([ERA5 model outputs](../modeling/era5_outputs.md#solar-photovoltaic-generation-time-series), [PVLib modeling](../modeling/pvlib/index.rst)) +* PV generation time-series ([MERRA2 only (legacy)](../legacy/merra2/merra2_outputs.md#pv-generation-time-series)) **Temperature** @@ -49,7 +52,7 @@ The following outputs are currently supported for climate data: **Aerosols** -* PM2.5 time series ([MERRA2 only](../datasets/merra2/merra2_outputs.md#pm25-time-series)) +* PM2.5 time series ([MERRA2 only (legacy)](../legacy/merra2/merra2_outputs.md#pm25-time-series)) ### Mask Specific diff --git a/environment.yaml b/environment.yaml index 28bf3a8f..e593976e 100644 --- a/environment.yaml +++ b/environment.yaml @@ -15,7 +15,6 @@ dependencies: - pandas>=0.22.0 - xarray>=0.11.2 - dask>=0.18.0 - - netcdf4 - rioxarray # Recommended for pandas and xarray diff --git a/pyproject.toml b/pyproject.toml index bab31149..b22e0cdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ "bottleneck>=1.3.6", "numexpr==2.10.1", "xarray>=2024.9.0", - "netcdf4>=1.7.1.post2", "boto3==1.26.46", "toolz>=0.12.1", "requests>=2.32.3", @@ -33,6 +32,7 @@ dependencies = [ "pvlib>=0.12.0", "timezonefinder>=6.5.9", "pyproj>=3.6.1", + "cdsapi>=0.7.5", ] requires-python = ">=3.10" readme = "README.md" @@ -41,7 +41,6 @@ license = {text = "GPLv3"} [project.optional-dependencies] download = [ - "cdsapi>=0.7.5", "herbie-data>=2025.5.0", ] notebook = [ @@ -56,6 +55,7 @@ docs = [ ] accelerate = [ "numba>=0.61.0", + "psutil>=5.9.0", ] [tool.uv] diff --git a/src/geodata/__init__.py b/src/geodata/__init__.py index a9ee1652..190d1669 100644 --- a/src/geodata/__init__.py +++ b/src/geodata/__init__.py @@ -16,12 +16,17 @@ from ._version import __version__ from .cutout import Cutout from .dataset import Dataset -from .mask import Mask +from typing import cast + +from . import mask as _mask_pkg from .plot import * # noqa: F403 from .model import * # noqa: F403 +Mask = cast(type, getattr(_mask_pkg, "Mask")) +XarrayMask = cast(type, getattr(_mask_pkg, "XarrayMask")) + __author__ = "Michael Davidson (UCSD), William Honaker" __copyright__ = "GNU GPL 3 license" -__all__ = ["Cutout", "Dataset", "Mask", "__version__"] +__all__ = ["Cutout", "Dataset", "Mask", "XarrayMask", "__version__"] diff --git a/src/geodata/cutout.py b/src/geodata/cutout.py index 507cf932..732540e3 100644 --- a/src/geodata/cutout.py +++ b/src/geodata/cutout.py @@ -20,13 +20,10 @@ """ import logging -from functools import partial from pathlib import Path -from typing import Literal, Optional, Union +from typing import Optional, Union import numpy as np -import pyproj -import shapely import xarray as xr from shapely.geometry import box from tqdm.auto import tqdm @@ -45,6 +42,7 @@ ) from .datasets._base import BaseDataset from .mask import Mask +from .mask.spatial import calc_grid_area, calc_shp_area, coarsen, ds_reformat_index from .preparation import ( cutout_get_meta, cutout_get_meta_view, @@ -517,145 +515,4 @@ def _convert_cutout( pv = pv -def ds_reformat_index(ds: xr.DataArray) -> xr.DataArray: - """Format the dataArray generated from the convert function. - - Args: - ds (xr.DataArray): dataArray generated from the convert function. - - Returns: - xr.DataArray: DataArray with lat and lon as dimensions. - """ - - if "lat" in ds.dims and "lon" in ds.dims: - return ds.sortby(["lat", "lon"]) - elif "lat" in ds.coords and "lon" in ds.coords: - return ( - ds.reset_coords(["lon", "lat"], drop=True) - .rename({"x": "lon", "y": "lat"}) - .sortby(["lat", "lon"]) - ) - return ds.rename({"x": "lon", "y": "lat"}).sortby(["lat", "lon"]) - - -def _find_intercept(list1, list2, start, threshold=0): - """Find_intercept is a helper function to find the best start point for doing coarsening - in order to make the coordinates of the coarsen as close to the target as possible. - """ - min_res = 0 - init = 0 - for i in range(len(list1) - start): - resid = ((list1[start + i] - list2[0]) % (list2[1] - list2[0])).values.tolist() - if i == 0: - init = resid - if resid <= threshold: - return i - if resid > min_res: - min_res = resid - else: - min_res = resid - break - if min_res == init: - return 0 - else: - return i - - -def coarsen(ori: xr.Dataset, tar: xr.Dataset, func: Literal["sum", "mean"] = "mean"): - """This function will reindex the original xarray dataset according to the coordiantes of the target. - There might be a bias for lattitudes and longitudes. The bias are normally within 0.01 degrees. - In order to not lose too much data, a threshold for bias in degree could be given. - When threshold = 0, it means that the function is going to find the best place with smallest bias. - - Args: - ori (xr.Dataset): The original xarray dataset. - tar (xr.Dataset): The target xarray dataset. - func (Literal['sum', 'mean']): The function to be used for reduction. Defaults to "mean". - - Returns: - xr.Dataset: The reindexed xarray dataset. - - Raises: - ValueError: reduction method can only be 'mean' or 'sum'. - """ - lat_multiple = round( - ((tar.lat[1] - tar.lat[0]) / (ori.lat[1] - ori.lat[0])).values.tolist() - ) - lon_multiple = round( - ((tar.lon[1] - tar.lon[0]) / (ori.lon[1] - ori.lon[0])).values.tolist() - ) - lat_start = _find_intercept(ori.lat, tar.lat, (lat_multiple - 1) // 2) - lon_start = _find_intercept(ori.lon, tar.lon, (lon_multiple - 1) // 2) - - if func == "mean": - _coarsen = ( - ori.isel(lat=slice(lat_start, None), lon=slice(lon_start, None)) - .coarsen( - dim={"lat": lat_multiple, "lon": lon_multiple}, - side={"lat": "left", "lon": "left"}, - boundary="pad", - ) - .mean() - ) - elif func == "sum": - _coarsen = ( - ori.isel(lat=slice(lat_start, None), lon=slice(lon_start, None)) - .coarsen( - dim={"lat": lat_multiple, "lon": lon_multiple}, - side={"lat": "left", "lon": "left"}, - boundary="pad", - ) - .sum() - ) - else: - raise ValueError("func can only be 'mean' or 'sum'") - - return _coarsen.reindex_like(tar, method="nearest") - - -def calc_grid_area(lis_lats_lons): - """Calculate area in km^2 for a grid cell given lats and lon border, with help from: - https://stackoverflow.com/questions/4681737/how-to-calculate-the-area-of-a-polygon-on-the-earths-surface-using-python - - """ - lons, lats = zip(*lis_lats_lons) - ll = list(set(lats))[::-1] - var = [] - for i in range(len(ll)): - var.append("lat_" + str(i + 1)) - st = "" - for v, l in zip(var, ll): # noqa: E741 - st = st + str(v) + "=" + str(l) + " " + "+" - st = ( - st - + "lat_0=" - + str(np.mean(ll)) - + " " - + "+" - + "lon_0" - + "=" - + str(np.mean(lons)) - ) - tx = "+proj=aea +" + st - pa = pyproj.Proj(tx) - - x, y = pa(lons, lats) - cop = {"type": "Polygon", "coordinates": [zip(x, y)]} - - return shapely.geometry.shape(cop).area / 1000000 - - -def calc_shp_area(shp, shp_projection="+proj=latlon"): - """calculate area in km^2 of the shapes for each shp object""" - temp_shape = shapely.ops.transform( - partial( - pyproj.transform, - pyproj.Proj(shp_projection), - pyproj.Proj(proj="aea", lat_1=shp.bounds[1], lat_2=shp.bounds[3]), - ), - shp, - ) - return temp_shape.area / 1000000 - - -__all__ = ["Cutout", "coarsen", "calc_grid_area", "calc_shp_area"] +__all__ = ["Cutout", "coarsen", "calc_grid_area", "calc_shp_area", "ds_reformat_index"] diff --git a/src/geodata/datasets/_base.py b/src/geodata/datasets/_base.py index 8ac18bd2..a7f867a4 100644 --- a/src/geodata/datasets/_base.py +++ b/src/geodata/datasets/_base.py @@ -307,17 +307,26 @@ def download(self, force: bool = False): if file.check(): logger.debug("Postprocessing %s", file.path) - ds = xr.open_dataset(file.path).chunk("auto") + # Check if this is a wind-solar dataset and use h5netcdf engine + # is_wind_solar = "wind_solar" in self.weather_config + # if is_wind_solar: + ds = xr.open_dataset(file.path, engine="h5netcdf").chunk("auto") + # else: + # ds = xr.open_dataset(file.path).chunk("auto") ds = self._rename_and_clean_coords(ds) ds = self._dataset_postprocess(ds) # xarray does not support overwriting files, so we must save the # dataset to a new file and then rename it backwards - ds.to_netcdf(file.path.with_stem(file.path.stem + "_postprocessed")) + postprocessed_path = file.path.with_stem(file.path.stem + "_postprocessed") + # if is_wind_solar: + ds.to_netcdf(postprocessed_path, engine="h5netcdf") + # else: + # ds.to_netcdf(postprocessed_path) ds.close() file.path.unlink() - file.path.with_stem(file.path.stem + "_postprocessed").rename(file.path) + postprocessed_path.rename(file.path) logger.info(f"Downloaded {self}") logger.info("Cleaning and renaming coordinates") diff --git a/src/geodata/datasets/era5/__init__.py b/src/geodata/datasets/era5/__init__.py index ca00fb2d..a1afc8b4 100644 --- a/src/geodata/datasets/era5/__init__.py +++ b/src/geodata/datasets/era5/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,6 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from . import hourly, monthly +from . import fixture, wind_3d, wind_solar -__all__ = ["hourly", "monthly"] +__all__ = ["fixture", "wind_3d", "wind_solar"] diff --git a/src/geodata/datasets/era5/_base.py b/src/geodata/datasets/era5/_base.py index ecc1ef11..78e0eb95 100644 --- a/src/geodata/datasets/era5/_base.py +++ b/src/geodata/datasets/era5/_base.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,12 +14,11 @@ # along with this program. If not, see . import logging -import os import numpy as np import xarray as xr -from ...types import CoordRange, PathLike +from geodata.types import CoordRange, PathLike from .._base import BaseDataset logger = logging.getLogger(__name__) @@ -141,23 +140,16 @@ def prepare_func( ys: slice, **kwargs, ): - """Prepare the dataset for a given year and month.""" - if isinstance(fn, str) and not os.path.exists(fn): - return - if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): - return - - with xr.open_dataset(fn) as ds: - logger.info("Opening %s", fn) - ds = _subset_x_y_era5(ds, xs, ys) - - # New ERA5 format for hourly datasets - # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 - # TODO: We can remove this if we refactor geodata's convert module in the future - if "valid_time" in ds.coords: - ds = ds.rename({"valid_time": "time"}) - - yield (year, month), ds + """Prepare the dataset for a given year and month. + + This method should be overridden by subclasses (e.g., ERA5Wind3DBaseDataset, + ERA5WindSolarBaseDataset) to provide dataset-specific preparation logic. + """ + raise NotImplementedError( + "prepare_func must be implemented by a subclass. " + "Use ERA5Wind3DBaseDataset or ERA5WindSolarBaseDataset, " + "or override this method in your subclass." + ) def _dataset_postprocess(self, ds, **kwargs): return super()._dataset_postprocess(ds, **kwargs) diff --git a/src/geodata/datasets/era5/fixture.py b/src/geodata/datasets/era5/fixture.py new file mode 100644 index 00000000..28f58d07 --- /dev/null +++ b/src/geodata/datasets/era5/fixture.py @@ -0,0 +1,145 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Offline ERA5 datasets backed by committed NetCDF files under ``tests/fixtures/``. + +On construction, small template files are **copied** into +``DATASET_ROOT_PATH / era5 / / …`` so paths stay compatible with +model code that uses :meth:`~geodata.model.results.BaseModelResult.ref_path`. + +Importing this module registers ``wind_3d_hourly_test`` and ``wind_solar_hourly_test`` +in :data:`geodata.datasets.registry`. +""" + +from __future__ import annotations + +import logging +import shutil +from pathlib import Path + +from geodata.config import DATASET_ROOT_PATH + +from .._base import AtomicDataset +from .wind_3d.hourly import ERA5Wind3DHourlyDataset +from .wind_solar.hourly import ERA5WindSolarHourlyDataset + +logger = logging.getLogger(__name__) + +# Paths must match tests/fixtures/era5//... +_FIXTURE_YEAR = 2016 +_FIXTURE_MONTH = 1 +_FIXTURE_DAY = 1 + + +def _resolve_fixture_root(config_dirname: str) -> Path: + """Return ``tests/fixtures/era5/`` by walking parents of this file. + + Works for editable installs where the repo contains ``tests/fixtures``. Wheel-only + installs without that tree raise ``FileNotFoundError``. + """ + here = Path(__file__).resolve() + for root in [here.parent, *here.parents]: + candidate = root / "tests" / "fixtures" / "era5" / config_dirname + if candidate.is_dir(): + return candidate + raise FileNotFoundError( + f"Could not find tests/fixtures/era5/{config_dirname} starting from {here}. " + "Offline fixture datasets need the repository tests/fixtures tree (e.g. editable install)." + ) + + +def _copy_fixture_into_storage(template_root: Path, storage_root: Path, relative: Path) -> None: + src = template_root / relative + if not src.is_file(): + raise FileNotFoundError(f"Expected fixture NetCDF at {src}") + dest = storage_root / relative + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + + +class ERA5Wind3DHourlyTestDataset(ERA5Wind3DHourlyDataset): + """Same schema as :class:`ERA5Wind3DHourlyDataset`, but points at a single local file. + + ``years`` / ``months`` passed to :meth:`__init__` do not expand the catalog; the + catalog is always the fixture for ``{_FIXTURE_YEAR}/{_FIXTURE_MONTH:02d}/{_FIXTURE_DAY:02d}.nc``. + """ + + weather_config = "wind_3d_hourly_test" + + def _extra_setup(self, **kwargs): + template_root = _resolve_fixture_root("wind_3d_hourly_test") + self.storage_root = DATASET_ROOT_PATH / self.module / self.weather_config + rel = ( + Path(str(_FIXTURE_YEAR)) + / f"{_FIXTURE_MONTH:02d}" + / f"{_FIXTURE_DAY:02d}.nc" + ) + _copy_fixture_into_storage(template_root, self.storage_root, rel) + + @property + def catalog(self) -> list[AtomicDataset]: + return [AtomicDataset(self, _FIXTURE_YEAR, _FIXTURE_MONTH, _FIXTURE_DAY)] + + def get_monthly_catalog(self, year: int, month: int) -> list[AtomicDataset]: + """Only the committed fixture day exists under ``ref_path``; do not list full month.""" + if not isinstance(year, int): + raise ValueError("year must be an integer") + if not isinstance(month, int): + raise ValueError("month must be an integer") + if not 1 <= month <= 12: + raise ValueError("month must be between 1 and 12") + if not self.years.start <= year <= self.years.stop: + raise ValueError( + f"year must be between {self.years.start} and {self.years.stop}" + ) + if not self.months.start <= month <= self.months.stop: + raise ValueError( + f"month must be between {self.months.start} and {self.months.stop}" + ) + if year == _FIXTURE_YEAR and month == _FIXTURE_MONTH: + return [AtomicDataset(self, year, month, _FIXTURE_DAY)] + return [] + + def _download_file(self, file: AtomicDataset): + raise RuntimeError( + f"{self.weather_config} uses committed fixtures under tests/fixtures; download is disabled." + ) + + +class ERA5WindSolarHourlyTestDataset(ERA5WindSolarHourlyDataset): + """Same schema as :class:`ERA5WindSolarHourlyDataset`, but points at one monthly fixture file.""" + + weather_config = "wind_solar_hourly_test" + + def _extra_setup(self, **kwargs): + template_root = _resolve_fixture_root("wind_solar_hourly_test") + self.storage_root = DATASET_ROOT_PATH / self.module / self.weather_config + rel = Path(str(_FIXTURE_YEAR)) / f"{_FIXTURE_MONTH:02d}.nc" + _copy_fixture_into_storage(template_root, self.storage_root, rel) + + @property + def catalog(self) -> list[AtomicDataset]: + return [AtomicDataset(self, _FIXTURE_YEAR, _FIXTURE_MONTH)] + + def _download_file(self, file: AtomicDataset): + raise RuntimeError( + f"{self.weather_config} uses committed fixtures under tests/fixtures; download is disabled." + ) + + +__all__ = [ + "ERA5Wind3DHourlyTestDataset", + "ERA5WindSolarHourlyTestDataset", +] diff --git a/src/geodata/datasets/era5/hourly/__init__.py b/src/geodata/datasets/era5/wind_3d/__init__.py similarity index 73% rename from src/geodata/datasets/era5/hourly/__init__.py rename to src/geodata/datasets/era5/wind_3d/__init__.py index b76666fc..c89ef813 100644 --- a/src/geodata/datasets/era5/hourly/__init__.py +++ b/src/geodata/datasets/era5/wind_3d/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from .wind_3d import ERA5Wind3DHourlyDataset -from .wind_solar import ERA5WindSolarHourlyDataset +from .hourly import ERA5Wind3DHourlyDataset + +__all__ = ["ERA5Wind3DHourlyDataset"] -__all__ = ["ERA5WindSolarHourlyDataset", "ERA5Wind3DHourlyDataset"] diff --git a/src/geodata/datasets/era5/wind_3d/_base.py b/src/geodata/datasets/era5/wind_3d/_base.py new file mode 100644 index 00000000..ddbcdf08 --- /dev/null +++ b/src/geodata/datasets/era5/wind_3d/_base.py @@ -0,0 +1,67 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os + +import xarray as xr + +from geodata.types import PathLike +from .._base import ERA5BaseDataset, _subset_x_y_era5 + +logger = logging.getLogger(__name__) + + +class ERA5Wind3DBaseDataset(ERA5BaseDataset): + """Base class for ERA5 3D wind datasets. + + This class provides the prepare_func implementation specific to wind_3d datasets, + which use model levels and the reanalysis-era5-complete product. + """ + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month. + + This implementation is specific to wind_3d datasets which: + - Use model levels (model_level coordinate) + - Download from reanalysis-era5-complete product + - Are stored as daily files + """ + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn, engine="h5netcdf") as ds: + logger.info("Opening %s", fn) + ds = _subset_x_y_era5(ds, xs, ys) + + # New ERA5 format for hourly datasets + # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 + # TODO: We can remove this if we refactor geodata's convert module in the future + if "valid_time" in ds.coords: + ds = ds.rename({"valid_time": "time"}) + + yield (year, month), ds + diff --git a/src/geodata/datasets/era5/hourly/wind_3d.py b/src/geodata/datasets/era5/wind_3d/hourly.py similarity index 84% rename from src/geodata/datasets/era5/hourly/wind_3d.py rename to src/geodata/datasets/era5/wind_3d/hourly.py index 95aeef4f..70585fa3 100644 --- a/src/geodata/datasets/era5/hourly/wind_3d.py +++ b/src/geodata/datasets/era5/wind_3d/hourly.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,6 +14,7 @@ # along with this program. If not, see . import logging +import os import pprint import tempfile from pathlib import Path @@ -21,12 +22,12 @@ import xarray as xr from ..._base import AtomicDataset -from .._base import ERA5BaseDataset +from ._base import ERA5Wind3DBaseDataset logger = logging.getLogger(__name__) -class ERA5Wind3DHourlyDataset(ERA5BaseDataset): +class ERA5Wind3DHourlyDataset(ERA5Wind3DBaseDataset): """ERA5Wind3DHourlyDataset is a class that handles the downloading, preprocessing, and storing of the ERA5 dataset for wind information. This dataset is stored in hourly intervals. @@ -83,6 +84,9 @@ def _download_file(self, file: AtomicDataset): try: _count += 1 full_result.download(save_path) + # Ensure file is fully written to disk before proceeding + with open(save_path, "rb") as f: + os.fsync(f.fileno()) logger.info("File downloaded: %s", save_path) return except Exception as e: @@ -98,19 +102,25 @@ def _download_file(self, file: AtomicDataset): try: _count += 1 full_result.download(tmpfile.name) + # Ensure file is fully written to disk before proceeding + os.fsync(tmpfile.fileno()) logger.info("File downloaded: %s", save_path) break except Exception as e: logger.error("Download failed: %s", e) if _count == 3: raise - with xr.open_dataset(tmpfile.name, chunks="auto") as ds: + with xr.open_dataset(tmpfile.name, chunks="auto", engine="h5netcdf") as ds: ds = ds.sel( longitude=slice(*sorted([self.bounds[0], self.bounds[2]])), latitude=slice( *sorted([self.bounds[1], self.bounds[3]], reverse=True) ), ) - ds.to_netcdf(save_path) + ds.to_netcdf(save_path, engine="h5netcdf") + # Ensure file is fully written to disk before proceeding + with open(save_path, "rb") as f: + os.fsync(f.fileno()) logger.info("File downloaded: %s", save_path) + diff --git a/src/geodata/datasets/era5/wind_solar/__init__.py b/src/geodata/datasets/era5/wind_solar/__init__.py new file mode 100644 index 00000000..407b1eb4 --- /dev/null +++ b/src/geodata/datasets/era5/wind_solar/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .hourly import ERA5WindSolarHourlyDataset +from .monthly import ERA5WindSolarMonthlyDataset + +__all__ = ["ERA5WindSolarHourlyDataset", "ERA5WindSolarMonthlyDataset"] + diff --git a/src/geodata/datasets/era5/wind_solar/_base.py b/src/geodata/datasets/era5/wind_solar/_base.py new file mode 100644 index 00000000..8bdf287a --- /dev/null +++ b/src/geodata/datasets/era5/wind_solar/_base.py @@ -0,0 +1,165 @@ +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os + +import xarray as xr +import numpy as np + +from geodata.types import PathLike +from .._base import ERA5BaseDataset, _subset_x_y_era5 + +logger = logging.getLogger(__name__) + +def _add_height(ds): + """Convert geopotential 'z' to geopotential height following [1] + + References + ---------- + [1] ERA5: surface elevation and orography, retrieved: 10.02.2019 + https://confluence.ecmwf.int/display/CKB/ERA5%3A+surface+elevation+and+orography + + """ + g0 = 9.80665 + z = ds["z"] + if "time" in z.coords: + z = z.isel(time=0, drop=True) + ds["height"] = z / g0 + ds = ds.drop("z") + return ds + +class ERA5WindSolarBaseDataset(ERA5BaseDataset): + """Base class for ERA5 wind and solar datasets. + + This class provides the prepare_func implementation specific to wind_solar datasets, + which use single-level data from the reanalysis-era5-single-levels product. + """ + + @classmethod + def transform_wind_solar_dataset(cls, ds: xr.Dataset) -> xr.Dataset: + """Transform raw ERA5 wind_solar dataset to standardized variable names and units. + + This method applies the transformations needed to convert raw ERA5 variables + to the standardized format used by models (e.g., pvlib). It can be called + directly on an already-opened dataset. + + Args: + ds: Raw ERA5 dataset with original variable names (fdir, tisr, t2m, u100, v100, etc.) + + Returns: + Transformed dataset with standardized variable names (influx_direct, influx_diffuse, + temperature, wnd100m, etc.) + """ + # Add height from geopotential if not already present + if "height" not in ds.data_vars and "z" in ds.data_vars: + ds = _add_height(ds) + + # Rename radiation variables + ds = ds.rename({"fdir": "influx_direct", "tisr": "influx_toa"}) + + # Calculate albedo and influx_diffuse + with np.errstate(divide="ignore", invalid="ignore"): + ds["albedo"] = ( + ((ds["ssrd"] - ds["ssr"]) / ds["ssrd"]) + .fillna(0.0) + .assign_attrs(units="(0 - 1)", long_name="Albedo") + ) + influx_diffuse = ds["ssrd"] - ds["influx_direct"] + influx_diffuse.attrs.update({ + "units": "J m**-2", + "long_name": "Surface diffuse solar radiation downwards" + }) + ds["influx_diffuse"] = influx_diffuse + ds = ds.drop(["ssrd", "ssr"]) + + # Convert from energy to power J m**-2 -> W m**-2 and clip negative fluxes + for a in ("influx_direct", "influx_diffuse", "influx_toa"): + ds[a] = ds[a].clip(min=0.0) / (60.0 * 60.0) + ds[a].attrs["units"] = "W m**-2" + + # Calculate wind speed from u and v components + wnd100m = np.sqrt(ds["u100"] ** 2 + ds["v100"] ** 2) + if isinstance(wnd100m, xr.DataArray): + wnd100m.attrs.update({ + "units": ds["u100"].attrs.get("units", ""), + "long_name": "100 metre wind speed" + }) + else: + # If it's a numpy array, convert to DataArray with attrs + wnd100m = xr.DataArray( + wnd100m, + coords=ds["u100"].coords, + dims=ds["u100"].dims, + attrs={ + "units": ds["u100"].attrs.get("units", ""), + "long_name": "100 metre wind speed" + } + ) + ds["wnd100m"] = wnd100m + ds = ds.drop(["u100", "v100"]) + + # Rename other variables + ds = ds.rename( + { + "ro": "runoff", + "t2m": "temperature", + "sp": "pressure", + "stl4": "soil temperature", + "fsr": "roughness", + "d2m": "dewpoint_temperature" + } + ) + + # New ERA5 format for hourly datasets + # See https://forum.ecmwf.int/t/new-time-format-in-era5-netcdf-files/3796 + # TODO: We can remove this if we refactor geodata's convert module in the future + if "valid_time" in ds.coords: + ds = ds.rename({"valid_time": "time"}) + + return ds + + @classmethod + def prepare_func( + cls, + fn: PathLike, + year: int, + month: int, + xs: slice, + ys: slice, + **kwargs, + ): + """Prepare the dataset for a given year and month. + + This implementation is specific to wind_solar datasets which: + - Use single-level data (no model levels) + - Download from reanalysis-era5-single-levels product + - Are stored as monthly files + """ + if isinstance(fn, str) and not os.path.exists(fn): + return + if isinstance(fn, list) and not all(os.path.isfile(f) for f in fn): + return + + with xr.open_dataset(fn) as ds: + logger.info("Opening %s", fn) + ds = _add_height(ds) + ds = _subset_x_y_era5(ds, xs, ys) + + # Use the shared transformation method + ds = cls.transform_wind_solar_dataset(ds) + + yield (year, month), ds + diff --git a/src/geodata/datasets/era5/hourly/wind_solar.py b/src/geodata/datasets/era5/wind_solar/hourly.py similarity index 92% rename from src/geodata/datasets/era5/hourly/wind_solar.py rename to src/geodata/datasets/era5/wind_solar/hourly.py index 8b139951..bf69fe5d 100644 --- a/src/geodata/datasets/era5/hourly/wind_solar.py +++ b/src/geodata/datasets/era5/wind_solar/hourly.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2024-2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -23,12 +23,12 @@ import xarray as xr from ..._base import AtomicDataset -from .._base import ERA5BaseDataset +from ._base import ERA5WindSolarBaseDataset logger = logging.getLogger(__name__) -class ERA5WindSolarHourlyDataset(ERA5BaseDataset): +class ERA5WindSolarHourlyDataset(ERA5WindSolarBaseDataset): """ERA5WindSolarHourlyDataset is a class that handles the downloading, preprocessing, and storing of the ERA5 dataset for wind and solar information. This dataset is stored in hourly intervals. @@ -81,13 +81,16 @@ def _download_file(self, file: AtomicDataset): month: int = file.month save_path: Path = file.path + # Limit to first 3 days when testing=True + max_day = 4 if self.testing else 32 + full_request = { "product_type": self.product_type, "format": "netcdf", "variable": list(self.variables.keys()), "year": year, "month": month, - "day": [f"{d:02d}" for d in range(1, 32)], + "day": [f"{d:02d}" for d in range(1, max_day)], "time": [f"{t:02d}:00" for t in range(0, 24)], } @@ -124,9 +127,10 @@ def _download_file(self, file: AtomicDataset): os.path.join(tempdir, f) for f in os.listdir(tempdir) if f.endswith(".nc") - ] + ], engine="h5netcdf" ) as ds: - ds.to_netcdf(save_path) + ds.to_netcdf(save_path, engine="h5netcdf") logger.info("Preprocessing complete with zipfile") logger.info("Successfully downloaded to %s", save_path) + diff --git a/src/geodata/datasets/era5/monthly/wind_solar.py b/src/geodata/datasets/era5/wind_solar/monthly.py similarity index 94% rename from src/geodata/datasets/era5/monthly/wind_solar.py rename to src/geodata/datasets/era5/wind_solar/monthly.py index 8a523b48..8b716c58 100644 --- a/src/geodata/datasets/era5/monthly/wind_solar.py +++ b/src/geodata/datasets/era5/wind_solar/monthly.py @@ -1,4 +1,4 @@ -# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD), Keyu Long (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -23,7 +23,7 @@ import xarray as xr from ..._base import AtomicDataset -from ..hourly.wind_solar import ERA5WindSolarHourlyDataset +from .hourly import ERA5WindSolarHourlyDataset logger = logging.getLogger(__name__) @@ -106,9 +106,10 @@ def _download_file(self, file: AtomicDataset): os.path.join(tempdir, f) for f in os.listdir(tempdir) if f.endswith(".nc") - ] + ], engine="h5netcdf" ) as ds: - ds.to_netcdf(save_path) + ds.to_netcdf(save_path, engine="h5netcdf") logger.info("Preprocessing complete with zipfile") logger.info("Successfully downloaded to %s", save_path) + diff --git a/src/geodata/datasets/hrrr/_base.py b/src/geodata/datasets/hrrr/_base.py index 1525c5d1..f0385d76 100644 --- a/src/geodata/datasets/hrrr/_base.py +++ b/src/geodata/datasets/hrrr/_base.py @@ -23,7 +23,7 @@ import pandas as pd import xarray as xr -from ...types import CoordRange +from geodata.types import CoordRange from .._base import BaseDataset logger = logging.getLogger(__name__) diff --git a/src/geodata/datasets/merra2/_base.py b/src/geodata/datasets/merra2/_base.py index d5fc7639..fdbd7a5d 100644 --- a/src/geodata/datasets/merra2/_base.py +++ b/src/geodata/datasets/merra2/_base.py @@ -24,7 +24,7 @@ import xarray as xr from tqdm.auto import tqdm -from ...types import CoordRange, PathLike +from geodata.types import CoordRange, PathLike from .._base import AtomicDataset, BaseDataset logger = logging.getLogger(__name__) diff --git a/src/geodata/mask.py b/src/geodata/mask.py index 1ce90885..c8f91eb8 100644 --- a/src/geodata/mask.py +++ b/src/geodata/mask.py @@ -144,7 +144,7 @@ def _add_layer( # replace layer by default if layer_name in self.layers: if replace is True: - self.layers[layer_name].close() + _close_dataset(self.layers[layer_name]) del self.layers[layer_name] # delete old layer from memory logger.info("Overwriting existing layer %s.", layer_name) else: @@ -231,7 +231,7 @@ def remove_layer(self, name: str): name (str): The name of the layer to be removed. """ if name in self.layers: - self.layers[name].close() + _close_dataset(self.layers[name]) del self.layers[name] else: raise KeyError(f"No layer name {name} found in the mask.") @@ -474,8 +474,8 @@ def merge_layer( merging_layers += list(temp_layers.values()) arr, aff = merge(merging_layers, method=_sum_method, **kwargs) - for layer in temp_layers.values(): - layer.close() + for layer in merging_layers: + _close_dataset(layer) else: raise ValueError(f"Method {method} is not supported.") @@ -491,13 +491,16 @@ def merge_layer( if attribute_save is True: if self.merged_mask: logger.info("Overwriting current merged_mask.") + _close_dataset(self.merged_mask) self.merged_mask = return_ras logger.info("Merged Mask saved as attribute 'merged_mask'.") + self.saved = False return return_ras def remove_merge_layer(self): """Remove the saved merged mask.""" + _close_dataset(self.merged_mask) self.merged_mask = None def add_shape_layer( @@ -684,7 +687,7 @@ def extract_shapes( return_shape[key] = raster if attribute_save: if key in self.shape_mask: - self.shape_mask[key].close() + _close_dataset(self.shape_mask[key]) logger.info( "[Overwritten] Extracted shape %s added to attribute 'shape_mask'.", key, @@ -714,7 +717,7 @@ def remove_shapes(self, names: Iterable[str]): for name in names: if name not in self.shape_mask.values(): raise KeyError(f"Shape mask {name} not found in the object.") - self.shape_mask[name].close() + _close_dataset(self.shape_mask[name]) del self.shape_mask[name] def load_merged_xr(self) -> xr.DataArray: @@ -774,14 +777,13 @@ def close_files(self): """Close all the opened rasters. This method will disable further save_mask() call.""" for layer in self.layers.values(): - layer.close() + _close_dataset(layer) - if self.merged_mask: - self.merged_mask.close() + _close_dataset(self.merged_mask) if self.shape_mask: for mask in self.shape_mask.values(): - mask.close() + _close_dataset(mask) def save_mask( self, @@ -991,6 +993,52 @@ def ras_to_xarr( return xarr +def _attach_memfile( + dataset: ras.DatasetReader, memfile: MemoryFile +) -> ras.DatasetReader: + """Pin ``memfile`` on ``dataset`` so in-memory GDAL paths stay valid.""" + dataset._geodata_memfile = memfile # type: ignore[attr-defined] + return dataset + + +def _close_dataset(dataset: ras.DatasetReader | None) -> None: + """Close a dataset and its pinned ``MemoryFile``, if any.""" + if dataset is None or dataset.closed: + return + memfile = getattr(dataset, "_geodata_memfile", None) + dataset.close() + if memfile is not None: + memfile.close() + + +def _open_memory_dataset( + arr: np.ndarray, + transform: ras.Affine, + *, + crs: str | ras.crs.CRS = "+proj=latlong", + compress: str = "lzw", + count: int = 1, +) -> ras.DatasetReader: + """Write ``arr`` to a GeoTIFF in memory and return an open reader.""" + memfile = MemoryFile() + with memfile.open( + driver="GTiff", + height=arr.shape[0], + width=arr.shape[1], + count=count, + dtype=arr.dtype, + compress=compress, + crs=crs, + transform=transform, + ) as dst: + if arr.ndim == 2: + dst.write(arr, 1) + else: + dst.write(arr) + dataset = memfile.open() + return _attach_memfile(dataset, memfile) + + def create_temp_tif( arr: np.ndarray, transform: ras.Affine, open_raster: bool = True ) -> ras.DatasetReader | str: @@ -1008,25 +1056,10 @@ def create_temp_tif( rasterio.DatasetReader: The temporary raster. """ - with MemoryFile() as memfile: - with ras.open( - memfile.name, - "w", - driver="GTiff", - height=arr.shape[0], - width=arr.shape[1], - count=1, - dtype=arr.dtype, - compress="lzw", - crs="+proj=latlong", - transform=transform, - ) as dst: - dst.write(arr, 1) - - if open_raster: - return ras.open(memfile.name) - - return memfile.name + dataset = _open_memory_dataset(arr, transform) + if open_raster: + return dataset + return dataset.name def save_opened_raster(raster: ras.DatasetReader, path: str): @@ -1038,7 +1071,7 @@ def save_opened_raster(raster: ras.DatasetReader, path: str): """ arr, transform = raster.read(1), raster.transform - raster.close() + _close_dataset(raster) save_raster(arr, transform, path) @@ -1095,21 +1128,20 @@ def crop_raster( (bounds[0], bounds[1]), (bounds[2], bounds[3]) ) - with MemoryFile() as memfile: - kwargs = raster.meta.copy() - kwargs.update( - { - "height": window.height, - "width": window.width, - "transform": ras.windows.transform(window, raster.transform), - } - ) - - with ras.open(memfile.name, "w", compress="lzw", **kwargs) as dst: - dst.write(raster.read(window=window)) - dst.close() - - return ras.open(memfile.name) + data = raster.read(window=window) + kwargs = raster.meta.copy() + kwargs.update( + { + "height": window.height, + "width": window.width, + "transform": ras.windows.transform(window, raster.transform), + } + ) + memfile = MemoryFile() + with memfile.open(compress="lzw", **kwargs) as dst: + dst.write(data) + dataset = memfile.open() + return _attach_memfile(dataset, memfile) def reproject_raster( @@ -1143,26 +1175,26 @@ def reproject_raster( # write it to another file: the CRS corrected one # rasterio.readthedocs.io/en/latest/topics/reproject.html - with MemoryFile() as memfile: - with ras.open(memfile.name, "w", compress="lzw", **kwargs) as dst: - for i in range(1, src.count + 1): - ras.warp.reproject( - source=ras.band(src, i), - destination=ras.band(dst, i), - src_transform=src.transform, - src_crs=src_crs, - dst_transform=transform, - dst_crs=dst_crs, - resampling=ras.warp.Resampling.nearest, - ) + memfile = MemoryFile() + with memfile.open(compress="lzw", **kwargs) as dst: + for i in range(1, src.count + 1): + ras.warp.reproject( + source=ras.band(src, i), + destination=ras.band(dst, i), + src_transform=src.transform, + src_crs=src_crs, + dst_transform=transform, + dst_crs=dst_crs, + resampling=ras.warp.Resampling.nearest, + ) - logger.info("Raster %s has been reprojected to %s CRS.", src.name, dst_crs) - return_ras = ras.open(memfile.name) + logger.info("Raster %s has been reprojected to %s CRS.", src.name, dst_crs) + return_ras = _attach_memfile(memfile.open(), memfile) - if trim: - return trim_raster(return_ras) + if trim: + return trim_raster(return_ras) - return return_ras + return return_ras def apply_fn_to_raster(raster: ras.DatasetReader, fn: callable): diff --git a/src/geodata/mask/__init__.py b/src/geodata/mask/__init__.py new file mode 100644 index 00000000..3f4102f2 --- /dev/null +++ b/src/geodata/mask/__init__.py @@ -0,0 +1,41 @@ +"""Mask package namespace. + +This package hosts mask-related modules (e.g. spatial helper utilities) while +preserving backward-compatible access to the legacy ``geodata.mask`` module API. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from .spatial import calc_grid_area, calc_shp_area, coarsen, ds_reformat_index +from .xarray_mask import XarrayMask + +_LEGACY_MODULE_PATH = Path(__file__).resolve().parent.parent / "mask.py" +_LEGACY_SPEC = importlib.util.spec_from_file_location( + "geodata._legacy_mask_module", _LEGACY_MODULE_PATH +) +if _LEGACY_SPEC is None or _LEGACY_SPEC.loader is None: + raise ImportError(f"Could not load legacy mask module from {_LEGACY_MODULE_PATH}") +_legacy_mask_module = importlib.util.module_from_spec(_LEGACY_SPEC) +_LEGACY_SPEC.loader.exec_module(_legacy_mask_module) + +# Re-export all public names from legacy ``mask.py``. +for _name in dir(_legacy_mask_module): + if _name.startswith("_"): + continue + globals()[_name] = getattr(_legacy_mask_module, _name) + +# Keep explicit access to phase-1 extracted helpers in this namespace. +globals().update( + { + "ds_reformat_index": ds_reformat_index, + "coarsen": coarsen, + "calc_grid_area": calc_grid_area, + "calc_shp_area": calc_shp_area, + "XarrayMask": XarrayMask, + } +) + +__all__ = [name for name in globals() if not name.startswith("_")] diff --git a/src/geodata/mask/spatial.py b/src/geodata/mask/spatial.py new file mode 100644 index 00000000..9c46544a --- /dev/null +++ b/src/geodata/mask/spatial.py @@ -0,0 +1,129 @@ +"""Shared spatial helper utilities for masking workflows.""" + +from functools import partial +from typing import Any, Literal, cast + +import numpy as np +import pyproj +import shapely +import xarray as xr +from shapely import ops + + +def ds_reformat_index(ds: xr.Dataset | xr.DataArray) -> xr.Dataset | xr.DataArray: + """Normalize data coordinates to sorted ``lat``/``lon``.""" + if "lat" in ds.dims and "lon" in ds.dims: + return ds.sortby(["lat", "lon"]) + if "lat" in ds.coords and "lon" in ds.coords: + return ( + ds.reset_coords(["lon", "lat"], drop=True) + .rename({"x": "lon", "y": "lat"}) + .sortby(["lat", "lon"]) + ) + return ds.rename({"x": "lon", "y": "lat"}).sortby(["lat", "lon"]) + + +def _find_intercept(list1, list2, start, threshold=0): + """Find best start offset for coarsening alignment.""" + min_res = 0 + init = 0 + i = 0 + for i in range(len(list1) - start): + resid = ((list1[start + i] - list2[0]) % (list2[1] - list2[0])).values.tolist() + if i == 0: + init = resid + if resid <= threshold: + return i + if resid > min_res: + min_res = resid + else: + min_res = resid + break + if min_res == init: + return 0 + return i + + +def coarsen( + ori: xr.Dataset | xr.DataArray, + tar: xr.Dataset | xr.DataArray, + func: Literal["sum", "mean"] = "mean", +): + """Reindex/coarsen ``ori`` according to target coordinates in ``tar``.""" + lat_multiple = round( + ((tar.lat[1] - tar.lat[0]) / (ori.lat[1] - ori.lat[0])).values.tolist() + ) + lon_multiple = round( + ((tar.lon[1] - tar.lon[0]) / (ori.lon[1] - ori.lon[0])).values.tolist() + ) + lat_start = _find_intercept(ori.lat, tar.lat, (lat_multiple - 1) // 2) + lon_start = _find_intercept(ori.lon, tar.lon, (lon_multiple - 1) // 2) + + if func == "mean": + coarsened = ( + ori.isel(lat=slice(lat_start, None), lon=slice(lon_start, None)) + .coarsen( + dim={"lat": lat_multiple, "lon": lon_multiple}, + side={"lat": "left", "lon": "left"}, + boundary="pad", + ) + ) + reduced = cast(Any, coarsened).mean() + elif func == "sum": + coarsened = ( + ori.isel(lat=slice(lat_start, None), lon=slice(lon_start, None)) + .coarsen( + dim={"lat": lat_multiple, "lon": lon_multiple}, + side={"lat": "left", "lon": "left"}, + boundary="pad", + ) + ) + reduced = cast(Any, coarsened).sum() + else: + raise ValueError("func can only be 'mean' or 'sum'") + + return reduced.reindex_like(tar, method="nearest") + + +def calc_grid_area(lis_lats_lons): + """Calculate area in km^2 for a grid cell defined by corner coordinates.""" + lons, lats = zip(*lis_lats_lons) + ll = list(set(lats))[::-1] + var = [] + for i in range(len(ll)): + var.append("lat_" + str(i + 1)) + st = "" + for v, l in zip(var, ll): # noqa: E741 + st = st + str(v) + "=" + str(l) + " " + "+" + st = ( + st + + "lat_0=" + + str(np.mean(ll)) + + " " + + "+" + + "lon_0" + + "=" + + str(np.mean(lons)) + ) + tx = "+proj=aea +" + st + pa = pyproj.Proj(tx) + + x, y = pa(lons, lats) + cop = {"type": "Polygon", "coordinates": [zip(x, y)]} + return shapely.geometry.shape(cop).area / 1000000 + + +def calc_shp_area(shp, shp_projection="+proj=latlon"): + """Calculate area in km^2 for a shape object.""" + temp_shape = ops.transform( + partial( + pyproj.transform, + pyproj.Proj(shp_projection), + pyproj.Proj(proj="aea", lat_1=shp.bounds[1], lat_2=shp.bounds[3]), + ), + shp, + ) + return temp_shape.area / 1000000 + + +__all__ = ["ds_reformat_index", "coarsen", "calc_grid_area", "calc_shp_area"] diff --git a/src/geodata/mask/xarray_mask.py b/src/geodata/mask/xarray_mask.py new file mode 100644 index 00000000..18e71d3f --- /dev/null +++ b/src/geodata/mask/xarray_mask.py @@ -0,0 +1,173 @@ +"""Xarray-native mask adapter for applying saved Mask objects to datasets.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal, cast + +import numpy as np +import xarray as xr + +from .spatial import calc_grid_area, coarsen, ds_reformat_index + + +def _ensure_dataset(data: xr.Dataset | xr.DataArray) -> xr.Dataset: + if isinstance(data, xr.Dataset): + return data + name = data.name or "value" + return data.to_dataset(name=name) + + +def _to_mask_2d(mask: xr.DataArray) -> xr.DataArray: + mask = mask.reset_coords(drop=True) + if "band" in mask.dims: + mask = mask.isel(band=0, drop=True) + return mask.transpose("lat", "lon") + + +@dataclass +class XarrayMask: + """Mask adapter that aligns saved mask rasters to a target xarray grid.""" + + grid: xr.Dataset + merged_mask: xr.DataArray | None = None + shape_masks: dict[str, xr.DataArray] = field(default_factory=dict) + + @classmethod + def from_mask( + cls, + mask, + grid: xr.Dataset | xr.DataArray, + include_merged: bool = True, + include_shapes: bool = True, + ) -> "XarrayMask": + grid_ds = ds_reformat_index(_ensure_dataset(grid)) + grid_ds = cast(xr.Dataset, grid_ds) + merged = None + shapes: dict[str, xr.DataArray] = {} + + if include_merged and mask.merged_mask: + merged = coarsen(mask.load_merged_xr(), grid_ds) + if include_shapes and mask.shape_mask: + shapes = {k: coarsen(v, grid_ds) for k, v in mask.load_shape_xr().items()} + + if merged is None and not shapes: + raise ValueError( + f"No mask found in {mask.name}. Please create a proper mask object first." + ) + + return cls(grid=grid_ds, merged_mask=merged, shape_masks=shapes) + + @classmethod + def from_name( + cls, + name: str, + grid: xr.Dataset | xr.DataArray, + mask_dir: str | None = None, + include_merged: bool = True, + include_shapes: bool = True, + ) -> "XarrayMask": + from geodata import Mask # lazy import to avoid circular imports + + if mask_dir is None: + from geodata import config + + mask = Mask.from_name(name, mask_dir=config.MASK_DIR) + else: + mask = Mask.from_name(name, mask_dir=mask_dir) + return cls.from_mask( + mask, + grid=grid, + include_merged=include_merged, + include_shapes=include_shapes, + ) + + @staticmethod + def compute_grid_area(grid: xr.Dataset | xr.DataArray) -> xr.DataArray: + xr_ds = ds_reformat_index(_ensure_dataset(grid)) + area_arr = np.zeros((xr_ds.lat.shape[0], xr_ds.lon.shape[0])) + lat_diff = np.abs((xr_ds.lat[1].values - xr_ds.lat[0].values)) + for i, lat in enumerate(xr_ds.lat.values): + lat_bottom = lat - lat_diff / 2 + lat_top = lat + lat_diff / 2 + area_arr[i] = np.round( + calc_grid_area( + [ + (xr_ds.lon.values[0], lat_top), + (xr_ds.lon.values[0], lat_bottom), + (xr_ds.lon.values[1], lat_bottom), + (xr_ds.lon.values[1], lat_top), + ] + ), + 2, + ) + return xr.DataArray( + area_arr, + dims=("lat", "lon"), + coords={"lat": xr_ds.lat.values, "lon": xr_ds.lon.values}, + name="area", + ) + + def _target_masks(self) -> dict[str, xr.DataArray]: + res: dict[str, xr.DataArray] = {} + if self.merged_mask is not None: + res["merged_mask"] = _to_mask_2d(self.merged_mask) + for key, value in self.shape_masks.items(): + res[key] = _to_mask_2d(value) + return res + + def attach( + self, dataset: xr.Dataset | xr.DataArray, include_area: bool = True + ) -> dict[str, xr.Dataset]: + ds = ds_reformat_index(_ensure_dataset(dataset)) + if "time" in ds.dims: + ds = ds.transpose("time", "lat", "lon") + + masks = self._target_masks() + if not masks: + raise ValueError("No masks available in XarrayMask.") + + area = self.compute_grid_area(self.grid) if include_area else None + out: dict[str, xr.Dataset] = {} + for key, mask in masks.items(): + cur = ds.assign({"mask": mask}) + if area is not None: + cur = cur.assign({"area": area}) + out[key] = cur + return out + + def apply( + self, + dataset: xr.Dataset | xr.DataArray, + mode: Literal["where", "multiply"] = "where", + include_area: bool = False, + ) -> dict[str, xr.Dataset]: + ds = ds_reformat_index(_ensure_dataset(dataset)) + if "time" in ds.dims: + ds = ds.transpose("time", "lat", "lon") + + masks = self._target_masks() + if not masks: + raise ValueError("No masks available in XarrayMask.") + if mode not in {"where", "multiply"}: + raise ValueError("mode can only be 'where' or 'multiply'") + + area = self.compute_grid_area(self.grid) if include_area else None + out: dict[str, xr.Dataset] = {} + for key, mask in masks.items(): + valid = mask > 0 + cur = ds.copy() + for var in list(cur.data_vars): + da = cur[var] + if "lat" in da.dims and "lon" in da.dims: + if mode == "where": + cur = cur.assign({var: cast(Any, da.where(valid))}) + else: + cur = cur.assign({var: cast(Any, da * valid)}) + if include_area and area is not None: + cur = cur.assign({"area": area}) + out[key] = cast(xr.Dataset, cur) + return out + + +__all__ = ["XarrayMask"] diff --git a/src/geodata/model/__init__.py b/src/geodata/model/__init__.py index 4a3ee665..c8ec0cf3 100644 --- a/src/geodata/model/__init__.py +++ b/src/geodata/model/__init__.py @@ -14,5 +14,6 @@ # along with this program. If not, see . from . import wind +from . import pvlib -__all__ = ["wind"] +__all__ = ["wind", "pvlib"] diff --git a/src/geodata/model/_base.py b/src/geodata/model/_base.py index a8a89bd7..15a05075 100644 --- a/src/geodata/model/_base.py +++ b/src/geodata/model/_base.py @@ -17,9 +17,12 @@ import abc import importlib.util import os +import platform import shutil -from typing import Optional +from collections.abc import Collection +from typing import ClassVar, Optional +import numpy as np import xarray as xr from tqdm.auto import tqdm @@ -29,16 +32,134 @@ from .results import DailyModelResult, MonthlyModelResult, ResultType if importlib.util.find_spec("h5netcdf") is not None: - XR_PARALLEL = True XR_ENGINE = "h5netcdf" + XR_PARALLEL_DEFAULT = True else: - XR_PARALLEL = False + XR_PARALLEL_DEFAULT = False XR_ENGINE = None logger.warning( "h5netcdf is not installed. Parallel reading of netCDF files will be disabled. " "This could have some performance implications." ) + +def _normalize_slice_for_sel(coord: xr.DataArray, s: slice) -> slice: + """Return a slice for ``.sel()`` that matches the coordinate direction. + + xarray's ``.sel(dim=slice(a, b))`` returns empty when the dimension is descending + (e.g. ERA5 latitude) or when the user passes ``slice(high, low)`` on an ascending + dimension. This helper interprets the slice as the inclusive logical range + ``[min(start, stop), max(start, stop)]`` and returns bounds in the order required + by ``.sel()`` for that coordinate's monotonic direction. + """ + if not isinstance(s, slice) or s.step not in (None, 1): + return s + if s.start is None or s.stop is None: + return s + lo, hi = min(s.start, s.stop), max(s.start, s.stop) + vals = np.asarray(coord.values).ravel() + if len(vals) < 2: + return slice(lo, hi) + descending = np.all(np.diff(vals) <= 0) + if descending: + return slice(hi, lo) + return slice(lo, hi) + + +def _is_in_dask_worker_on_linux() -> bool: + """Check if we're running in a Dask worker process on Linux. + + Returns: + bool: True if we're in a Dask worker on Linux, False otherwise. + """ + if platform.system() != "Linux": + return False + + try: + from dask.distributed import get_worker + try: + get_worker() + return True + except ValueError: + # Not in a worker process + return False + except ImportError: + # dask.distributed not available + return False + + +def _is_dask_using_processes_on_linux() -> bool: + """Check if Dask is being used with processes on Linux. + + Returns: + bool: True if Dask is using processes on Linux, False otherwise. + + Note: + This checks if there's an active Dask client using processes. + When Dask uses processes, h5netcdf has issues with HDF5 dimension scales. + """ + if platform.system() != "Linux": + return False + + try: + from dask.distributed import get_client, get_worker + try: + get_client() + # Check if we're in a worker (which means processes are being used) + try: + get_worker() + return True + except ValueError: + # Not in a worker, but check if client exists and might use processes + # We can't easily detect this from the main process, so we'll be conservative + # and assume processes might be used if a client exists + # The actual check will happen in workers via _is_in_dask_worker_on_linux + return False + except ValueError: + # No active client + return False + except ImportError: + # dask.distributed not available + return False + + +def _get_xr_engine() -> str | None: + """Get the appropriate xarray engine to use for opening NetCDF files. + + Returns: + str | None: The engine name to use, or None for default. + """ + logger.debug(f"_get_xr_engine: Returning engine {XR_ENGINE}") + return XR_ENGINE + + +def _should_use_parallel_reading() -> bool: + """Determine if parallel reading should be used for xarray open_mfdataset. + + Returns: + bool: True if parallel reading should be used, False otherwise. + + Note: + Parallel reading is disabled when Dask is using processes on Linux, + as h5netcdf has issues with HDF5 dimension scales in that case. + """ + if not XR_PARALLEL_DEFAULT: + logger.debug("_should_use_parallel_reading: XR_PARALLEL_DEFAULT is False, returning False") + return False + + in_worker = _is_in_dask_worker_on_linux() + using_processes = _is_dask_using_processes_on_linux() + + if in_worker or using_processes: + logger.info( + f"_should_use_parallel_reading: Disabling parallel reading " + f"(in_worker={in_worker}, using_processes={using_processes})" + ) + return False + + logger.debug(f"_should_use_parallel_reading: Returning {XR_PARALLEL_DEFAULT}") + return XR_PARALLEL_DEFAULT + # Parse the MAX_WORKERS environment variable if present MAX_WORKERS = os.getenv("MAX_WORKERS") if MAX_WORKERS is not None: @@ -62,7 +183,7 @@ class BaseModel(abc.ABC): **kwargs: Additional keyword arguments to pass to the model. """ - SUPPORTED_WEATHER_DATA_CONFIGS: tuple[str] + SUPPORTED_WEATHER_DATA_CONFIGS: ClassVar[Collection[str]] def __init__(self, source: BaseDataset, **kwargs): if source.weather_config not in self.SUPPORTED_WEATHER_DATA_CONFIGS: @@ -189,12 +310,27 @@ def estimate( results = self.get_result_year_month(years, months) files = sum([result.files for result in results], []) - params = xr.open_mfdataset(files, engine=XR_ENGINE, parallel=XR_PARALLEL) + engine = _get_xr_engine() + parallel = _should_use_parallel_reading() + logger.info( + f"estimate: Opening {len(files)} files with engine={engine}, parallel={parallel}" + ) + params = xr.open_mfdataset(files, engine=engine, parallel=parallel) if xs is not None: - params = params.sel(x=xs) + x_slice = ( + _normalize_slice_for_sel(params.coords["x"], xs) + if "x" in params.coords + else xs + ) + params = params.sel(x=x_slice) if ys is not None: - params = params.sel(y=ys) + y_slice = ( + _normalize_slice_for_sel(params.coords["y"], ys) + if "y" in params.coords + else ys + ) + params = params.sel(y=y_slice) output = self._estimate_dataset(params, **kwargs) params.close() @@ -235,8 +371,15 @@ def prepare(self, force: bool = False): shutil.rmtree(result.path, ignore_errors=True) result.path.mkdir(parents=True, exist_ok=True) + engine = _get_xr_engine() + parallel = _should_use_parallel_reading() + logger.info( + f"prepare: Opening {len(result.ref_files)} files with engine={engine}, parallel={parallel}" + ) with xr.open_mfdataset( - result.ref_files, engine=XR_ENGINE, parallel=XR_PARALLEL + result.ref_files, + engine=engine, + parallel=parallel, ) as ds: prepared_ds = self._prepare_dataset(ds) result.register(prepared_ds) diff --git a/src/geodata/datasets/era5/monthly/__init__.py b/src/geodata/model/pvlib/__init__.py similarity index 80% rename from src/geodata/datasets/era5/monthly/__init__.py rename to src/geodata/model/pvlib/__init__.py index 98d2dfe8..68ad363a 100644 --- a/src/geodata/datasets/era5/monthly/__init__.py +++ b/src/geodata/model/pvlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 Michael Davidson (UCSD), Xiqiang Liu (UCSD) +# Copyright 2023 Michael Davidson (UCSD), Xiqiang Liu (UCSD) # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -13,6 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from .wind_solar import ERA5WindSolarMonthlyDataset +from ._base import Pvlib -__all__ = ["ERA5WindSolarMonthlyDataset"] +__all__ = ["Pvlib"] \ No newline at end of file diff --git a/src/geodata/model/pvlib/_base.py b/src/geodata/model/pvlib/_base.py new file mode 100644 index 00000000..4a77ebae --- /dev/null +++ b/src/geodata/model/pvlib/_base.py @@ -0,0 +1,1114 @@ +# Copyright 2016-2017 Gorm Andresen (Aarhus University), Jonas Hoersch (FIAS), Tom Brown (FIAS) +# Copyright 2020 Michael Davidson (UCSD), William Honaker, Jiahe Feng (UCSD), Yuanbo Shi +# Copyright 2023-2024 Xiqiang Liu, 2025 Keyu Long + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +GEODATA + +Geospatial Data Collection and "Pre-Analysis" Tools + +TODO: Documentation here + +""" +import os +import platform +import pandas as pd +import xarray as xr +import time +from multiprocessing import Pool, Manager, cpu_count as mp_cpu_count +from pvlib import pvsystem +from pvlib.location import Location +from pvlib.modelchain import ModelChain +from timezonefinder import TimezoneFinder + +from .._base import BaseModel, _normalize_slice_for_sel, _should_use_parallel_reading +from geodata.logging import logger +from .calculations import calculate_pvlib_solarposition, calculate_ghi, calculate_relative_humidity, calculate_precipitable_water, convert_kelvin_to_celsius +from tqdm.auto import tqdm + + +class ModelChainConfig: + """ + Defines pvlib ModelChain parameters as a class that + can be passed to one or more instances of pvlib_model(). + Allows user to reuse a common set of ModelChain parameters across multiple + PVSystems or even multiple cutouts. + + Parameters + ---------- + clearsky_model : string, default 'ineichen' + Specifies the clear-sky model. Passed to location.get_clearsky. + Only used when DNI is not found in the weather inputs. + transposition_model : string, default 'haydavies' + Specifies the transposition model. Passed to system.get_irradiance. + solar_position_method : string, default 'nrel_numpy' + Specifies the method for calculating solar positions. Passed to location.get_solarposition. + airmass_model : string, default 'kastenyoung1989' + Specifies the airmass model. Passed to location.get_airmass. + dc_model : string or function, optional + Specifies the DC model. Valid strings are 'sapm', 'desoto', 'cec', 'pvsyst', 'pvwatts'. + If not specified, the model will be inferred from the parameters of system.arrays[i].module_parameters. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + ac_model : string or function, optional + Specifies the AC model. Valid strings are 'sandia', 'adr', 'pvwatts'. + If not specified, the model will be inferred from the parameters of system.inverter_parameters. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + aoi_model : string or function, optional + Specifies the angle of incidence (AOI) model. Valid strings are 'physical', 'ashrae', 'sapm', 'martin_ruiz', + 'interp', 'no_loss'. If not specified, the model will be inferred from the parameters of + system.arrays[i].module_parameters. A user-defined function may also be provided, + with the ModelChain instance passed as the first argument. + spectral_model : string or function, optional + Specifies the spectral model. Valid strings are 'sapm', 'first_solar', 'no_loss'. + If not specified, the model will be inferred from the parameters of system.arrays[i].module_parameters. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + temperature_model : string or function, optional + Specifies the temperature model. Valid strings are 'sapm', 'pvsyst', 'faiman', 'fuentes', 'noct_sam'. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + dc_ohmic_model : string or function, default 'no_loss' + Specifies the DC ohmic loss model. Valid strings are 'dc_ohms_from_percent', 'no_loss'. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + losses_model : string or function, default 'no_loss' + Specifies the losses model. Valid strings are 'pvwatts', 'no_loss'. + A user-defined function may also be provided, with the ModelChain instance passed as the first argument. + name : string, optional + Specifies the name of the ModelChain instance. + + For full documentation, see: + - pvlib.modelchain.ModelChain(): + https://pvlib-python.readthedocs.io/en/stable/reference/generated/pvlib.modelchain.ModelChain.html + + """ + def __init__( + self, + clearsky_model='ineichen', + transposition_model='haydavies', + solar_position_method='nrel_numpy', + airmass_model='kastenyoung1989', + dc_model=None, + ac_model=None, + aoi_model=None, + spectral_model=None, + temperature_model=None, + dc_ohmic_model='no_loss', + losses_model='no_loss', + name=None + ): + self.clearsky_model = clearsky_model + self.transposition_model = transposition_model + self.solar_position_method = solar_position_method + self.airmass_model = airmass_model + self.dc_model = dc_model + self.ac_model = ac_model + self.aoi_model = aoi_model + self.spectral_model = spectral_model + self.temperature_model = temperature_model + self.dc_ohmic_model = dc_ohmic_model + self.losses_model = losses_model + self.name = name + + def model_chain_to_kwargs(self): + return self.__dict__ + + +def _detect_available_cpus() -> int: + """ + Detect the number of available CPUs using multiple methods. + + Tries multiple detection methods in order: + 1. SLURM environment variables (if in SLURM job) - authoritative for HPC clusters + 2. psutil (if available) - most reliable, respects CPU affinity + 3. Linux cgroups v2 (if available) - respects container limits + 4. Linux cgroups v1 (if available) - respects container limits + 5. multiprocessing.cpu_count() - standard library fallback + 6. os.cpu_count() - last resort + 7. Defaults to 1 if all methods fail + + Returns: + int: Number of available CPUs (at least 1) + """ + detected_cpus = None + method_used = None + + # Method 1: Try SLURM environment variables (for HPC clusters) + # SLURM is authoritative when present, so check this first + try: + # Check if we're in a SLURM job + if os.getenv("SLURM_JOB_ID") is not None: + # Try SLURM_CPUS_PER_TASK first (most common and reliable) + slurm_cpus_per_task = os.getenv("SLURM_CPUS_PER_TASK") + if slurm_cpus_per_task is not None: + try: + detected_cpus = int(slurm_cpus_per_task) + if detected_cpus > 0: + method_used = "SLURM_CPUS_PER_TASK" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using SLURM_CPUS_PER_TASK={slurm_cpus_per_task}" + ) + # SLURM is authoritative - return immediately + logger.debug( + f"_detect_available_cpus: Successfully detected {detected_cpus} CPU(s) " + f"using method: {method_used}" + ) + return detected_cpus + except (ValueError, TypeError): + logger.debug( + f"_detect_available_cpus: SLURM_CPUS_PER_TASK={slurm_cpus_per_task} " + f"is not a valid integer, trying other SLURM variables" + ) + + # If SLURM_CPUS_PER_TASK not available, try SLURM_JOB_CPUS_PER_NODE + if detected_cpus is None: + slurm_job_cpus = os.getenv("SLURM_JOB_CPUS_PER_NODE") + if slurm_job_cpus is not None: + try: + # SLURM_JOB_CPUS_PER_NODE can be a comma-separated list for multi-node jobs + # Take the first value (current node) + cpus_str = slurm_job_cpus.split(',')[0] + detected_cpus = int(cpus_str) + if detected_cpus > 0: + method_used = "SLURM_JOB_CPUS_PER_NODE" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using SLURM_JOB_CPUS_PER_NODE={slurm_job_cpus} " + f"(using first node value)" + ) + # SLURM is authoritative - return immediately + logger.debug( + f"_detect_available_cpus: Successfully detected {detected_cpus} CPU(s) " + f"using method: {method_used}" + ) + return detected_cpus + except (ValueError, TypeError, IndexError): + logger.debug( + f"_detect_available_cpus: SLURM_JOB_CPUS_PER_NODE={slurm_job_cpus} " + f"could not be parsed, trying other methods" + ) + + # If still not found, try SLURM_CPUS_ON_NODE (but this is less reliable) + # as it shows CPUs on node, not necessarily allocated to job + if detected_cpus is None: + slurm_cpus_on_node = os.getenv("SLURM_CPUS_ON_NODE") + if slurm_cpus_on_node is not None: + try: + # Can be a comma-separated list for multi-node jobs + cpus_str = slurm_cpus_on_node.split(',')[0] + detected_cpus = int(cpus_str) + if detected_cpus > 0: + method_used = "SLURM_CPUS_ON_NODE" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using SLURM_CPUS_ON_NODE={slurm_cpus_on_node} " + f"(using first node value). Note: This may not reflect " + f"actual CPU allocation to the job." + ) + # SLURM is authoritative - return immediately + logger.debug( + f"_detect_available_cpus: Successfully detected {detected_cpus} CPU(s) " + f"using method: {method_used}" + ) + return detected_cpus + except (ValueError, TypeError, IndexError): + logger.debug( + f"_detect_available_cpus: SLURM_CPUS_ON_NODE={slurm_cpus_on_node} " + f"could not be parsed, trying other methods" + ) + + if detected_cpus is None: + logger.debug( + "_detect_available_cpus: Running in SLURM job (SLURM_JOB_ID present) " + "but no usable CPU count variables found. Trying other detection methods." + ) + except Exception as e: + logger.debug(f"_detect_available_cpus: SLURM detection failed: {e}, trying other methods") + + # Method 2: Try psutil (most reliable, respects CPU affinity and cgroups) + if detected_cpus is None: + try: + import psutil + detected_cpus = psutil.cpu_count(logical=False) # Physical cores first + if detected_cpus is None or detected_cpus == 0: + detected_cpus = psutil.cpu_count(logical=True) # Fallback to logical cores + if detected_cpus is not None and detected_cpus > 0: + method_used = "psutil" + logger.debug(f"_detect_available_cpus: Detected {detected_cpus} CPU(s) using psutil") + except ImportError: + logger.debug("_detect_available_cpus: psutil not available, trying other methods") + except Exception as e: + logger.debug(f"_detect_available_cpus: psutil failed: {e}, trying other methods") + + # Method 3: Try Linux cgroups v2 (for containers) + if detected_cpus is None and platform.system() == "Linux": + try: + # Check cgroup v2 cpu.max (format: "max" or "quota period") + cgroup_path = "/sys/fs/cgroup/cpu.max" + if os.path.exists(cgroup_path): + with open(cgroup_path, 'r') as f: + content = f.read().strip() + if content != "max": + parts = content.split() + if len(parts) == 2: + quota = int(parts[0]) + period = int(parts[1]) + if quota > 0 and period > 0: + detected_cpus = max(1, int(quota / period)) + method_used = "cgroups_v2" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using cgroups v2 (quota={quota}, period={period})" + ) + except Exception as e: + logger.debug(f"_detect_available_cpus: cgroups v2 check failed: {e}") + + # Method 4: Try Linux cgroups v1 (for containers) + if detected_cpus is None: + try: + # Check cgroup v1 cpu.cfs_quota_us and cpu.cfs_period_us + quota_path = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + period_path = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + if os.path.exists(quota_path) and os.path.exists(period_path): + with open(quota_path, 'r') as f: + quota = int(f.read().strip()) + with open(period_path, 'r') as f: + period = int(f.read().strip()) + if quota > 0 and period > 0: + detected_cpus = max(1, int(quota / period)) + method_used = "cgroups_v1" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using cgroups v1 (quota={quota}, period={period})" + ) + except Exception as e: + logger.debug(f"_detect_available_cpus: cgroups v1 check failed: {e}") + + # Method 5: Try multiprocessing.cpu_count() + if detected_cpus is None: + try: + detected_cpus = mp_cpu_count() + if detected_cpus is not None and detected_cpus > 0: + method_used = "multiprocessing.cpu_count()" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) " + f"using multiprocessing.cpu_count()" + ) + except Exception as e: + logger.debug(f"_detect_available_cpus: multiprocessing.cpu_count() failed: {e}") + + # Method 6: Try os.cpu_count() as last resort + if detected_cpus is None: + try: + detected_cpus = os.cpu_count() + if detected_cpus is not None and detected_cpus > 0: + method_used = "os.cpu_count()" + logger.debug( + f"_detect_available_cpus: Detected {detected_cpus} CPU(s) using os.cpu_count()" + ) + except Exception as e: + logger.debug(f"_detect_available_cpus: os.cpu_count() failed: {e}") + + # Final fallback: default to 1 + if detected_cpus is None or detected_cpus <= 0: + detected_cpus = 1 + method_used = "default_fallback" + logger.debug( + "_detect_available_cpus: All detection methods failed. " + "Defaulting to 1 CPU and logging debug message." + ) + logger.debug( + "_detect_available_cpus: This may indicate the program is running in a restricted " + "environment (container, cgroup limits, or CPU affinity restrictions)." + ) + else: + logger.debug( + f"_detect_available_cpus: Successfully detected {detected_cpus} CPU(s) " + f"using method: {method_used}" + ) + + return detected_cpus + + +def _process_single_coordinate(args): + """ + Helper function to process a single coordinate for multiprocessing. + + This function must be at module level to be picklable for multiprocessing. + + Args: + args: Tuple containing: + - coord: Tuple of (y, x) coordinates + - weather_data: DataFrame with weather data + - system: PVSystem object + - model_chain_kwargs: Dictionary of ModelChain configuration + - ptc: Module PTC value + - n_mods: Number of modules per string + - progress_dict: Shared dictionary for progress tracking (optional) + - coord_index: Index of this coordinate in the total list + - total_coords: Total number of coordinates to process + - compact_output: Whether to keep only ac/pv in output + + Returns: + Tuple of (coord, subset_df) where subset_df contains the processed data + """ + ( + (y, x), + weather_data, + system, + model_chain_kwargs, + ptc, + n_mods, + progress_dict, + coord_index, + total_coords, + compact_output, + ) = args + + try: + # Extract subset for this coordinate + subset = weather_data.loc[(slice(None), y, x), :].reset_index(['x', 'y']) + + # Get timezone + tz_str = TimezoneFinder().timezone_at(lat=y, lng=x) + if tz_str is None: + raise ValueError(f"Timezone not found for coordinates ({y}, {x})") + + # Create location + location = Location(latitude=y, longitude=x, tz=tz_str) # type: ignore[arg-type] + + # Create and run ModelChain + mc = ModelChain( + system, + location, + **model_chain_kwargs + ) + mc.run_model(subset) + + # Calculate outputs + subset['ac'] = mc.results.ac + subset.loc[subset['ac'] < 0, 'ac'] = 0 + subset['pv'] = subset['ac'] / (ptc * n_mods) + + # Update progress if progress_dict is provided + if progress_dict is not None: + with progress_dict['lock']: + progress_dict['completed'] += 1 + completed = progress_dict['completed'] + elapsed = time.time() - progress_dict['start_time'] + + # Log progress periodically + log_interval = max(1, min(100, total_coords // 10)) + if completed % log_interval == 0 or completed == total_coords: + avg_time_per_coord = elapsed / completed if completed > 0 else 0 + remaining_coords = total_coords - completed + eta = avg_time_per_coord * remaining_coords + progress_dict['last_log'] = { + 'completed': completed, + 'total': total_coords, + 'coord': (y, x), + 'elapsed': elapsed, + 'avg_time': avg_time_per_coord, + 'eta': eta + } + progress_dict['should_log'] = True + + # Re-pack results into a MultiIndex so that + # xr.Dataset.from_dataframe() reconstructs x and y as dimensions. + # + # `subset` is currently indexed only by `time` (x/y were reset into columns), + # which would otherwise cause the output to have only `time` as a coordinate. + if compact_output: + subset_out = subset[['ac', 'pv']].copy() + else: + subset_out = subset.copy() + subset_out = subset_out.assign(y=y, x=x) + subset_out = subset_out.reset_index() + + # After reset_index(), the time column name can vary (e.g. 'time' vs 'index'). + if subset.index.name is None: + subset_out = subset_out.rename(columns={'index': 'time'}) + elif subset.index.name != 'time': + subset_out = subset_out.rename(columns={subset.index.name: 'time'}) + + subset_out = subset_out.set_index(['time', 'x', 'y']) + return (y, x), subset_out + + except Exception as e: + logger.error(f"Error processing coordinate ({y}, {x}): {str(e)}") + raise + + +class Pvlib(BaseModel): + """The pvlib model""" + @property + def type(self) -> str: + return "pvlib" + + SUPPORTED_WEATHER_DATA_CONFIGS = ("wind_solar_hourly", "wind_solar_hourly_test") + + @property + def prepared(self) -> bool: + """This model does not need to be prepared""" + return True + + def prepare(self, force: bool = False): + """Skip preparation - this model doesn't need it.""" + logger.info("This model does not require preparation. Skipping.") + return + + def init_model_config( + self, + clearsky_model='ineichen', + transposition_model='haydavies', + solar_position_method='nrel_numpy', + airmass_model='kastenyoung1989', + dc_model=None, + ac_model=None, + aoi_model=None, + spectral_model=None, + temperature_model=None, + dc_ohmic_model='no_loss', + losses_model='no_loss', + name=None + ): + self.config = ModelChainConfig( + clearsky_model= clearsky_model, + transposition_model= transposition_model, + solar_position_method= solar_position_method, + airmass_model= airmass_model, + dc_model= dc_model, + ac_model= ac_model, + aoi_model= aoi_model, + spectral_model= spectral_model, + temperature_model= temperature_model, + dc_ohmic_model= dc_ohmic_model, + losses_model= losses_model, + name= name + ) + + def retrieve_sam(self, samfile, path=None): + """ + Wrapper for pvlib.pvsystem.retrieve_sam(). Retrieves latest module + and inverter info from a file bundled with pvlib, a path or a + URL (like SAM’s website), and returns it as a Pandas DataFrame. + + Supported databases: + - CEC module database + - Sandia Module database + - CEC Inverter database + - Anton Driesse Inverter database + + Parameters + ---------- + name : string + Use one of the following strings to retrieve a database bundled with pvlib: + - ’CECMod’ - returns the CEC module database + - ’CECInverter’ - returns the CEC Inverter database + - ’SandiaInverter’ - returns the CEC Inverter database + (CEC is only current inverter db available; tag kept for backwards compatibility) + - ’SandiaMod’ - returns the Sandia Module database + - ’ADRInverter’ - returns the ADR Inverter database + + Optional Parameters + ---------- + path : string + Path to a CSV file or a URL. + + Returns: DataFrame + + See also: + - pvlib.pvsystem.retrieve_sam(): + https://pvlib-python.readthedocs.io/en/stable/reference/generated/pvlib.pvsystem.retrieve_sam.html + + """ + return pvsystem.retrieve_sam(name=samfile, path=path) + + def init_pv_system(self, *args, **kwargs): + """ + Wrapper for pvlib.pvsystem.PVSystem(). + The PVSystem class defines a standard set of PV system attributes + and modeling functions. This class describes the collection and + interactions of PV system components rather than an installed system + on the ground. It is typically used in combination with Location + and ModelChain objects. + + The class supports basic system topologies consisting of: + - N total modules arranged in series (modules_per_string=N, strings_per_inverter=1). + - M total modules arranged in parallel (modules_per_string=1, strings_per_inverter=M). + - NxM total modules arranged in M strings of N modules each + (modules_per_string=N, strings_per_inverter=M). + + For full documentation, see: https://pvlib-python.readthedocs.io/en/stable/reference/generated/pvlib.pvsystem.PVSystem.html + + Parameters + ---------- + arrays : array (optional) + An Array or list of arrays that are part of the system. + See pvlib documentation for full description. + surface_tilt : float + Surface tilt angles in decimal degrees. The tilt angle is + defined as degrees from horizontal (e.g. surface facing up = 0, + surface facing horizon = 90). + surface_azimuth : float + Azimuth angle of the module surface. North=0, East=90, South=180, West=270. + albedo : float + Ground surface albedo. If not supplied, then surface_type is used to look up + a value in pvlib.albedo.SURFACE_ALBEDOS. If surface_type is also not supplied + then a ground surface albedo of 0.25 is used. + surface_type : string + The ground surface type. See pvlib.albedo.SURFACE_ALBEDOS for valid values. + module : string + The model name of the modules. May be used to look up the module_parameters dictionary via some other method. + module_type : string + Describes the module’s construction. Valid strings are ‘glass_polymer’ and ‘glass_glass’. + Used for cell and module temperature calculations. + module_parameters : dict + Module parameters as defined by the SAPM, CEC, or other. + temperature_model_parameters : dict + Temperature model parameters as required by one of the models in pvlib.temperature (excluding poa_global, temp_air and wind_speed). + modules_per_string : int, float + See system topology discussion above. + strings_per_inverter : int, float + See system topology discussion above. + inverter : string + The model name of the inverters. May be used to look up the inverter_parameters dictionary via some other method. + inverter_parameters : dict + Inverter parameters as defined by the SAPM, CEC, or other. + racking_model : string + Valid strings are ‘open_rack’, ‘close_mount’, and ‘insulated_back’. + Used to identify a parameter set for the SAPM cell temperature model. + losses_parameters : dict + Losses parameters as defined by PVWatts or other. + name : string (optional) + + """ + self.pv_system = pvsystem.PVSystem(*args, **kwargs) + + + def _estimate_dataset(self, params: xr.Dataset, **kwargs) -> xr.Dataset | xr.DataArray: # type: ignore[override] + """Estimate PV output from prepared dataset. + + Args: + params: Dataset (already filtered by years/months/xs/ys from BaseModel) + **kwargs: Additional parameters (not used currently, but available) + + Returns: + Dataset with AC power and PV capacity (returns Dataset, but BaseModel expects DataArray) + """ + + compact_output = kwargs.get("compact_output", True) + result = self._pvlib_model( + params, + self.pv_system, + self.config, + compact_output=compact_output, + ) + return result + + def estimate(self, + years: slice | None = None, + months: slice | None = None, + xs: slice | None = None, + ys: slice | None = None, + compact_output: bool = True, + **kwargs, + ) -> xr.DataArray: + """Get pvlib model results. + + This method processes data month-by-month to avoid memory issues with large datasets. + Results from each month are concatenated along the time dimension. + + Args: + years: Year range (slice) + months: Month range (slice) + xs: X-coordinate range (slice) + ys: Y-coordinate range (slice) + compact_output: If True (default), return only `ac` and `pv` + as data variables. If False, keep full per-coordinate output. + **kwargs: Additional parameters + + Returns: + Dataset with AC power and PV capacity, concatenated across all months + """ + if getattr(self, 'pv_system', None) is None: + raise ValueError("pv_system is not initialized. Call init_pv_system() first.") + if getattr(self, 'config', None) is None: + raise ValueError("model_config is not initialized. Call init_model_config() first.") + + # Get result objects for the requested time range + if years is None and months is None: + results = self.flattened_results + elif months is None: + # If years specified but months not, use all months + if years is None: + results = self.flattened_results + else: + results = self.get_result_year_month(years, slice(1, 13)) + else: + # Both years and months specified + if years is None: + # If only months specified, need to get all years + # Use the source dataset's year range + years = self.source.years + results = self.get_result_year_month(years, months) + + if not results: + raise ValueError("No results found for the specified year/month range.") + + # Process month-by-month to manage memory + logger.info( + f"Processing {len(results)} month(s) month-by-month to manage memory usage" + ) + + engine = "h5netcdf" + parallel = _should_use_parallel_reading() + + monthly_results = [] + + for result in tqdm(results, desc="Processing months", unit="month"): + # Load only this month's raw data files + ref_files = result.ref_files + + if not ref_files: + logger.warning( + f"No files found for {result.year:04d}-{result.month:02d}, skipping." + ) + continue + + logger.debug( + f"Loading {len(ref_files)} file(s) for {result.year:04d}-{result.month:02d} " + f"with engine={engine}, parallel={parallel}" + ) + + # Open this month's dataset + with xr.open_mfdataset( + ref_files, + engine=engine, + parallel=parallel, + ) as params: + # Rename coordinates to x/y if they use longitude/latitude naming + # This must happen before spatial filtering + # Check dimensions first (for .sel() to work), then coordinates + rename_dict = {} + if "longitude" in params.dims: + rename_dict["longitude"] = "x" + elif "lon" in params.dims: + rename_dict["lon"] = "x" + if "latitude" in params.dims: + rename_dict["latitude"] = "y" + elif "lat" in params.dims: + rename_dict["lat"] = "y" + + if rename_dict: + params = params.rename(rename_dict) + + # Apply spatial filtering if specified. + # Normalize slice order for descending coordinates (e.g. ERA5 latitude); + # otherwise .sel() returns empty. + if xs is not None: + x_slice = _normalize_slice_for_sel(params.coords["x"], xs) if "x" in params.coords else xs + params = params.sel(x=x_slice) + if ys is not None: + y_slice = _normalize_slice_for_sel(params.coords["y"], ys) if "y" in params.coords else ys + params = params.sel(y=y_slice) + + # Transform raw dataset to standardized format + # This applies the same transformations as prepare_func + # (renames variables, calculates derived quantities, etc.) + dataset_cls = type(self.source) + if hasattr(dataset_cls, 'transform_wind_solar_dataset'): + # Call the classmethod to transform the dataset + params = dataset_cls.transform_wind_solar_dataset(params) # type: ignore[attr-defined] + else: + logger.warning( + "Dataset does not have transform_wind_solar_dataset method. " + "Assuming data is already in the correct format." + ) + + # Process this month's data + monthly_output = self._estimate_dataset( + params, + compact_output=compact_output, + **kwargs, + ) + + # Store the result (will concatenate later) + monthly_results.append(monthly_output) + + if not monthly_results: + raise ValueError("No data was successfully processed for the specified range.") + + # Concatenate all monthly results along the time dimension + logger.info(f"Concatenating {len(monthly_results)} month(s) of results") + + # Ensure all datasets have compatible coordinates + # Sort by time to ensure proper ordering + combined_result = xr.concat(monthly_results, dim='time') + + # Sort by time to ensure chronological order + if 'time' in combined_result.coords: + combined_result = combined_result.sortby('time') + + # Standardize output dimension order across models: + # `("time", "x", "y")`. + desired_order = ("time", "x", "y") + if all(d in combined_result.dims for d in desired_order): + combined_result = combined_result.transpose(*desired_order) + return combined_result + + def _prepare_pvlib_ds(self, ds: xr.Dataset, *varnames: str) -> xr.Dataset: + """ + Prepares an `xarray.Dataset` from a geodata `cutout` class for use in model simulations using `pvlib`. + This function extracts specified variables from the `cutout` dataset, calculates additional parameters + like global horizontal irradiance (GHI), precipitable water, and solar position, and renames fields to + align with expected inputs. + + PARALLELIZATION OPTIONS: + ------------------------ + This function processes the entire dataset at once using vectorized operations (xarray/numpy). + Most operations are already parallelized at the numpy level (BLAS/MKL). + + Potential optimizations: + 1. If dataset is very large, consider chunking by coordinates and processing in parallel + 2. The solar position calculation (calculate_pvlib_solarposition) could be parallelized + across time steps if it's not already vectorized + 3. Consider using dask arrays for lazy evaluation if memory is a concern + + Requires a cutout with the following variables: + + - **influx_diffuse** (*float*) - Diffuse horizontal irradiance. + - **influx_direct** (*float*) - Direct normal irradiance. + - **dewpoint_temperature** (*float*) - Dewpoint temperature in Celsius. + - **temperature** (*float*) - Air temperature in Celsius. + - **wnd100m** (*float*) - Wind speed at 100m. + + Outputs an `xarray.Dataset` with the following variables: + + - **dhi** (*float*) - Diffuse horizontal irradiance. + - **dni** (*float*) - Direct normal irradiance. + - **ghi** (*float*) - Global horizontal irradiance (calculated via :code:`_calculate_ghi()`). + - **temp_air** (*float*) - Air temperature in Celsius. + - **wind_speed** (*float*) - Wind speed at 100m. + - **precipitable_water** (*float*) - Precipitable water (calculated via :code:`_calculate_precipitable_water()`). + + Parameters + ---------- + ds : xarray.Dataset + Must contain following variables: influx_diffuse, influx_direct, + dewpoint_temperature, temperature, wnd100m. + varnames : string + String values representing names of required variables. + + Returns + ------- + weather_data : `xarray.Dataset` + Dataset containing necessary variables to run `pvlib` model simulations. + + """ + prepare_start = time.time() + + if varnames: + # Check which variables are actually available + available_vars = [v for v in varnames if v in ds.data_vars] + missing_vars = [v for v in varnames if v not in ds.data_vars] + if missing_vars: + logger.warning(f"Missing variables: {missing_vars}. Available: {list(ds.data_vars.keys())}") + if available_vars: + ds = ds[available_vars] + else: + logger.error(f"None of the requested variables {varnames} are available in dataset") + raise KeyError(f"None of the requested variables {varnames} are available. Available variables: {list(ds.data_vars.keys())}") + + temperature_celsius = convert_kelvin_to_celsius(ds.temperature) + + relative_humidity = calculate_relative_humidity( + temperature_celsius, + convert_kelvin_to_celsius(ds.dewpoint_temperature), + # convert_kelvin_to_celsius(ds.d2m), + ) + + precipitable_water = calculate_precipitable_water( + temperature_celsius, + relative_humidity + ) + + sp = calculate_pvlib_solarposition(ds) + ghi = calculate_ghi(ds, sp['zenith']) + + ds = ( + ds + .assign( + ghi=ghi, + temperature=temperature_celsius, + precipitable_water=precipitable_water + ) + .rename({ + 'influx_diffuse': 'dhi', + 'influx_direct': 'dni', + 'temperature': 'temp_air', + 'wnd100m': 'wind_speed' + }) + ) + + result = ds[[ + "dhi", + "dni", + "ghi", + "temp_air", + "wind_speed", + "precipitable_water" + ]] + + prepare_time = time.time() - prepare_start + logger.info(f"_prepare_pvlib_ds: {prepare_time:.2f}s") + + return result + + def _pvlib_model( + self, + ds: xr.Dataset, + system: pvsystem.PVSystem, + model_chain_config: ModelChainConfig, + vars: list[str] = ["influx_diffuse", "influx_direct", "dewpoint_temperature", "temperature", "wnd100m"], + n_jobs: int | None = None, + compact_output: bool = True, + ) -> xr.Dataset: + + """ + Applies a `pvlib` model using :code:`pvlib.modelchain.ModelChain()` across all unique coordinates + represented in a `geodata` cutout. This function prepares input weather data, initializes the + `pvlib` model, and runs simulations for each set of coordinates, outputting an xarray dataset + containing all simulation results. + + PARALLELIZATION OPTIONS: + ------------------------ + This function processes coordinates sequentially, which is the main bottleneck. + Recommended parallelization approaches: + + Option 1: multiprocessing.Pool (Recommended for CPU-bound tasks) + ---------------------------------------- + - Use multiprocessing.Pool to process coordinates in parallel + - Each worker processes a subset of coordinates independently + - Pros: True parallelism, good for CPU-bound pvlib calculations + - Cons: Requires pickling system/model_chain_config objects, higher memory usage + + Option 2: concurrent.futures.ThreadPoolExecutor + ------------------------------------------------ + - Use threads for I/O-bound operations (if any) + - Less overhead than multiprocessing + - Pros: Lower memory overhead, faster startup + - Cons: Limited by GIL for CPU-bound tasks (pvlib is CPU-bound, so not ideal) + + Option 3: concurrent.futures.ProcessPoolExecutor + ------------------------------------------------ + - Similar to multiprocessing.Pool but with a simpler API + - Pros: Cleaner API, better error handling + - Cons: Similar to Option 1 + + Option 4: joblib.Parallel + -------------------------- + - High-level parallel processing library + - Pros: Simple API, good progress reporting, handles pickling well + - Cons: Additional dependency + + Option 5: Dask (for distributed computing) + ------------------------------------------- + - For very large datasets across multiple machines + - Pros: Scales to clusters, handles memory efficiently + - Cons: More complex setup, overhead for small datasets + + Implementation suggestion: + - Create a helper function: _process_single_coordinate(y, x, weather_data, system, model_chain_config, ptc, n_mods) + - Use multiprocessing.Pool.map() or ProcessPoolExecutor.map() to parallelize + - Consider chunking coordinates into batches to balance load + - Use n_jobs parameter to control parallelism (default: os.cpu_count()) + + Requires a cutout with the following variables: + + - **influx_diffuse** (*float*) - Diffuse horizontal irradiance. + - **influx_direct** (*float*) - Direct normal irradiance. + - **dewpoint_temperature** (*float*) - Dewpoint temperature in Celsius. + - **temperature** (*float*) - Air temperature in Celsius. + - **wnd100m** (*float*) - Wind speed at 100m. + + Outputs an `xarray.Dataset` containing: + + - **ac** (*float*) - AC photovoltaic output (W). + - **pv** (*float*) - Photovoltaic capacity. + + Parameters + ---------- + cutout : geodata **cutout** class + Cutout generated by the `geodata` library, based on the ERA5 dataset. + Must contain the required meteorological variables. + system : pvlib **PVSystem** class + The photovoltaic system to be simulated. Generated by :code:`geodata.pvlib.pv_system()` + model_chain_config : `ModelChainConfig` + Configuration object for :code:`pvlib.modelchain.ModelChain()` with model parameters. + vars : list of str, optional + List of variable names required for simulation. Defaults to: + ['influx_diffuse', 'influx_direct', 'dewpoint_temperature', 'temperature', 'wnd100m']. + compact_output : bool, optional + If True (default), keep only `ac` and `pv` data variables in the + final output. If False, keep the full per-coordinate output. + + Returns + ------- + xr.Dataset + Dataset containing ac power output and pv capacity across all coordinates in the cutout. + + """ + ptc = system.arrays[0].module_parameters['PTC'] + n_mods = system.arrays[0].modules_per_string + + weather_data = self._prepare_pvlib_ds(ds, *vars).to_dataframe() + unique_coords = weather_data.index.droplevel('time').drop_duplicates() + total_coords = len(unique_coords) + + # Determine number of workers using robust CPU detection + if n_jobs is None: + n_jobs = _detect_available_cpus() + logger.debug( + f"_pvlib_model: Auto-detected {n_jobs} available CPU(s) for parallel processing" + ) + else: + logger.debug( + f"_pvlib_model: Using user-specified n_jobs={n_jobs} for parallel processing" + ) + + # Ensure n_jobs is valid: at least 1, and not more than total coordinates + n_jobs = max(1, min(n_jobs, total_coords)) + + if n_jobs == 1: + logger.debug( + "_pvlib_model: Using sequential processing (n_jobs=1). " + "This may be due to: only 1 coordinate, CPU detection returned 1, " + "or user specified n_jobs=1" + ) + + logger.info( + f"Processing {total_coords} coordinate(s) using {n_jobs} worker process(es)" + ) + + # Prepare arguments for parallel processing + model_chain_kwargs = model_chain_config.model_chain_to_kwargs() + + # Create shared progress tracking dictionary + manager = Manager() + progress_dict = manager.dict() + progress_dict['completed'] = 0 + progress_dict['start_time'] = time.time() + progress_dict['should_log'] = False + progress_dict['last_log'] = None + progress_dict['lock'] = manager.Lock() + + # Prepare arguments for each coordinate + process_args = [ + ( + (y, x), + weather_data, + system, + model_chain_kwargs, + ptc, + n_mods, + progress_dict, + idx, + total_coords, + compact_output, + ) + for idx, (y, x) in enumerate(unique_coords, 1) + ] + + coord_start_time = time.time() + coord_subsets = [] + + # Process coordinates in parallel + if n_jobs == 1: + # Sequential processing (useful for debugging or when only 1 coordinate) + logger.debug("Using sequential processing (n_jobs=1)") + for args in process_args: + (y, x), subset = _process_single_coordinate(args) + coord_subsets.append(subset) + + # Log progress + idx = args[7] # coord_index + if idx % max(1, min(100, total_coords // 10)) == 0 or idx == total_coords: + elapsed_total = time.time() - coord_start_time + avg_time_per_coord = elapsed_total / idx + remaining_coords = total_coords - idx + eta = avg_time_per_coord * remaining_coords + logger.debug( + f"Processed coordinate {idx}/{total_coords} ({y:.2f}, {x:.2f}): " + f"Avg: {avg_time_per_coord:.2f}s/coord | " + f"ETA: {eta:.1f}s" + ) + else: + # Parallel processing with progress tracking + logger.debug(f"Using parallel processing with {n_jobs} workers") + + # Start a thread to monitor progress + import threading + stop_progress_thread = threading.Event() + + def progress_monitor(): + """Monitor progress and log updates""" + last_logged = 0 + while not stop_progress_thread.is_set(): + time.sleep(0.5) # Check every 0.5 seconds + if progress_dict.get('should_log', False): + with progress_dict['lock']: + if progress_dict.get('should_log', False): + log_info = progress_dict.get('last_log') + if log_info and log_info['completed'] > last_logged: + logger.debug( + f"Processed coordinate {log_info['completed']}/{log_info['total']} " + f"({log_info['coord'][0]:.2f}, {log_info['coord'][1]:.2f}): " + f"Avg: {log_info['avg_time']:.2f}s/coord | " + f"ETA: {log_info['eta']:.1f}s" + ) + last_logged = log_info['completed'] + progress_dict['should_log'] = False + + progress_thread = threading.Thread(target=progress_monitor, daemon=True) + progress_thread.start() + + try: + with Pool(processes=n_jobs) as pool: + results = pool.map(_process_single_coordinate, process_args) + + # Extract subsets from results + coord_subsets = [subset for (y, x), subset in results] + + finally: + stop_progress_thread.set() + progress_thread.join(timeout=1.0) + + elapsed_total = time.time() - coord_start_time + logger.info( + f"Completed processing {total_coords} coordinate(s) in {elapsed_total:.2f}s " + f"({elapsed_total/total_coords:.2f}s per coordinate on average)" + ) + + weather_data_final = pd.concat(coord_subsets).sort_index() + + out = xr.Dataset.from_dataframe(weather_data_final) + desired_order = ("time", "x", "y") + if all(d in out.dims for d in desired_order): + out = out.transpose(*desired_order) + return out + + def _prepare_dataset(self, source: xr.Dataset) -> xr.Dataset: + """This will never be called, but must be implemented (abstract method).""" + raise NotImplementedError("This model does not use _prepare_dataset") \ No newline at end of file diff --git a/src/geodata/model/pvlib/calculations.py b/src/geodata/model/pvlib/calculations.py new file mode 100644 index 00000000..80779d8f --- /dev/null +++ b/src/geodata/model/pvlib/calculations.py @@ -0,0 +1,250 @@ +import numpy as np +import pandas as pd +import xarray as xr + +from pvlib.atmosphere import gueymard94_pw +from pvlib.solarposition import get_solarposition + + + +def calculate_pvlib_solarposition(ds: xr.Dataset) -> pd.DataFrame: + """ + Wrapper for :code:`pvlib.solarposition.get_solarposition()`. + Allows for vectorized calculation of solar position across an xarray dataset. + The solar zenith angle is a required input for :code:`_calculate_ghi()`. + + For full documentation on how :code:`pvlib.solarposition.get_solarposition()` calculates precipitable water, + see: `the pvlib API reference for pvlib.solarposition.get_solarposition() `. + + Parameters + ---------- + ds : xarray dataset + An xarray dataset containing series for both influx diffuse (dhi) and influx direct (dni). + zenith : numeric + Zenith angle of the sun in degrees, as calculated by :code:`_calculate_pvlib_solarposition()`. + + Returns + ------- + solarposition : dataframe + Dataframe containing solar zenith angle for a given time and set of coordinates. + """ + nt, ny, nx = ds.sizes['time'], ds.sizes['y'], ds.sizes['x'] + time_expanded = np.broadcast_to(ds.time.values[:, None, None], (nt, ny, nx)).ravel() + yy, xx = np.meshgrid(ds.y, ds.x, indexing="ij") + x_expanded = np.tile(xx.ravel(), nt) + y_expanded = np.tile(yy.ravel(), nt) + solarposition = get_solarposition(time_expanded, y_expanded, x_expanded) # might return a pandas DataFrame or a NDArray + multi_index = pd.MultiIndex.from_arrays([time_expanded, y_expanded, x_expanded], names=['time', 'y', 'x']) + if isinstance(solarposition, pd.DataFrame): + solarposition = solarposition.set_index(multi_index) + else: + solarposition = pd.DataFrame(solarposition) + solarposition.index = multi_index + return solarposition + +def calculate_ghi( + ds: xr.Dataset, + zenith: pd.Series +) -> xr.DataArray: + """ + Calculates global horizontal irradiance (ghi) from data arrays representing influx diffuse (dhi) and influx direct (dni) + Negative values are clipped. Calculated using the formula: + + .. math:: + + GHI = DHI + DNI * cos(Z) + + where Z representst the solar zenith as calculated by :code:`calculate_pvlib_solarposition()`. + + Parameters + ---------- + ds : xarray dataset + An xarray dataset containing series for both influx diffuse (dhi) and influx direct (dni). + zenith : numeric + Zenith angle of the sun in degrees, as calculated by :code:`_calculate_pvlib_solarposition()`. + + Returns + ------- + ghi : numeric + Global horizontal irradiance (ghi) [W m**-2]. + + """ + # Convert zenith to numpy array if it's a pandas Series or xarray DataArray + if isinstance(zenith, (pd.Series, xr.DataArray)): + zenith_vals = zenith.values # type: ignore[union-attr] + else: + zenith_vals = zenith + # Ensure zenith_vals is a numpy array for type checking + zenith_vals = np.asarray(zenith_vals) + + dhi = ds.influx_diffuse.values.ravel() + dni = ds.influx_direct.values.ravel() + + if zenith_vals.size == 0: + x_coord = ds.coords.get("x") + if x_coord is None: + x_coord = ds.coords.get("longitude") if "longitude" in ds.coords else ds.coords.get("lon") + y_coord = ds.coords.get("y") + if y_coord is None: + y_coord = ds.coords.get("latitude") if "latitude" in ds.coords else ds.coords.get("lat") + x_vals = np.asarray(x_coord.values) if x_coord is not None else np.array([]) + y_vals = np.asarray(y_coord.values) if y_coord is not None else np.array([]) + time_size = ds.sizes.get("time", 0) + + def _fmt_coord(arr: np.ndarray, max_show: int = 20) -> str: + if len(arr) == 0: + return "[]" + if len(arr) <= max_show: + return str(arr.tolist()) + return f"[{arr.min():g}..{arr.max():g}] (length={len(arr)})" + + raise ValueError( + "Cannot calculate GHI: dataset has no data points. " + "This typically occurs when xs/ys slices do not overlap with the dataset's " + "coordinates, or when the loaded dataset is empty. " + f"Dataset dimensions: time={time_size}, x={_fmt_coord(x_vals)}, " + f"y={_fmt_coord(y_vals)}. " + "Check that your xs and ys values (in degrees) overlap with these coordinate ranges." + ) + + # TODO: check if zenith is in degrees or radians and convert to radians if needed + # it is processed from get_solarposition() + # if zenith is in degrees, convert to radians + if np.max(zenith_vals) > np.pi * 2: + zenith_vals = np.deg2rad(zenith_vals) + + ghi = np.clip( + dhi + dni * np.cos(zenith_vals), + 0, + np.inf # `np.Inf` was removed in the NumPy 2.0 release. + ) + + reshaped_ghi = ghi.reshape( + ds.sizes['time'], + ds.sizes['y'], + ds.sizes['x'] + ) + + ghi = xr.DataArray( + reshaped_ghi, + dims=("time", "y", "x"), + coords={ + "time": ds['time'].values, + "y": ds['y'].values, + "x": ds['x'].values + }, + name="ghi" + ) + + ghi.name = "ghi" + ghi.attrs["units"] = "W m**-2" + ghi.attrs["description"] = "Ghi calculated from influx diffuse (dhi) and influx direct (dni)." + return ghi + +def calculate_relative_humidity( + temperature: xr.DataArray, + dewpoint_temperature: xr.DataArray +) -> xr.DataArray: + """ + Calculates relative humidity based on air temperature and dewpoint temperature. + Needed in order to calculate precipitable water using pvlib's :code:`gueymard94_pw()` function. + + Relative humidity is calculated using a version of the + August-Roche-Magnus equation as follows: + + .. math:: + + RH = 100 \cdot \frac{{\exp\left(\frac{{17.625 \cdot TD}}{{243.04 + TD}}\right)}}{{\exp\left(\frac{{17.625 \cdot T}}{{243.04 + T}}\right)}} + + where, RH is % relative humidity, TD is dew-point temperature (celsius), and T is air temperature (celsius).[#1]_ [#2]_ + + Parameters + ---------- + temperature : numeric + Ambient air temperature at the surface. [C] + dewpoint_temperature : numeric + Dewpoint temperature at the surface. [C] + + Returns + ------- + relative_humidity : numeric + Percent relative humidity. [%] + + References + ---------- + .. [#1] `United States Environmental Protection Agency. Hydrologic Micro Services. Meteorology - Humidity. `_ + + .. [#2] `University of Miami. Calculate Temperature, Dewpoint, or Relative Humidity. ` + + """ + relative_humidity = 100 * ( + np.exp((17.625 * dewpoint_temperature) / (243.04 + dewpoint_temperature)) / + np.exp((17.625 * temperature) / (243.04 + temperature)) + ) + + # Ensure result is xarray DataArray (arithmetic operations preserve xarray types) + if not isinstance(relative_humidity, xr.DataArray): + relative_humidity = xr.DataArray(relative_humidity) + + relative_humidity.name = "relative_humidity" + relative_humidity.attrs["units"] = "%" + relative_humidity.attrs["description"] = "Relative humidity, calculated using temperature and dewpoint temperature." + + return relative_humidity + +def calculate_precipitable_water( + temperature: xr.DataArray, + relative_humidity: xr.DataArray +) -> xr.DataArray: + """ + Calculates precipitable water (cm) from ambient air temperature (C) and relative humidity (%) using + :code:`pvlib.atmosphere.gueymard94_pw()`. + + Precipitable water (cm) is a required input for models using CEC modules from :code:`pvlib`. + For full documentation on how :code:`pvlib.atmosphere.gueymard94_pw()` calculates precipitable water, + see: `the pvlib API reference for pvlib.atmosphere.gueymard94_pw() `. + + Parameters + ---------- + temperature : numeric + Ambient air temperature at the surface. [C] + relative_humidity : numeric + Percent relative humidity. [%] + + Returns + ------- + precipitable_water : numeric + Precipitable water (cm) calculated from ambient air temperature (C) and relative humidity (%). [cm] + + """ + # Use xarray's apply_ufunc to preserve DataArray type when calling external function + precipitable_water = xr.apply_ufunc( + gueymard94_pw, + temperature, + relative_humidity, + dask="allowed", + output_dtypes=[float] + ) + precipitable_water.name = "precipitable_water" + precipitable_water.attrs["units"] = "cm" + precipitable_water.attrs["description"] = "Precipitable water (cm) calculated from ambient air temperature (C) and relative humidity (%)." + + return precipitable_water + +def convert_kelvin_to_celsius( + ds: xr.DataArray +) -> xr.DataArray: + """ + Converts a temperature in Kelvin to a temperature in Celsius. + + Parameters + ---------- + ds : numeric or xarray.DataArray + A temperature in Kelvin [K]. + + Returns + ------- + temperature : numeric or xarray.DataArray + A temperature in Celsius [C]. + """ + return ds - 273.15 \ No newline at end of file diff --git a/src/geodata/model/results/daily.py b/src/geodata/model/results/daily.py index 1a60d0cf..2fb35745 100644 --- a/src/geodata/model/results/daily.py +++ b/src/geodata/model/results/daily.py @@ -115,9 +115,11 @@ def register(self, dataset: xr.Dataset): logger.debug("Saving model results to %s", self.path) - from .._base import XR_ENGINE + from .._base import _get_xr_engine - xr.save_mfdataset(datasets, paths, engine=XR_ENGINE) + engine = _get_xr_engine() + logger.info(f"register: Saving {len(paths)} files with engine={engine}") + xr.save_mfdataset(datasets, paths, engine=engine) # Write the hash file for integrity checking with ThreadPoolExecutor() as executor: diff --git a/src/geodata/model/results/monthly.py b/src/geodata/model/results/monthly.py index b8648ffa..3089258b 100644 --- a/src/geodata/model/results/monthly.py +++ b/src/geodata/model/results/monthly.py @@ -44,7 +44,11 @@ def _check_prepared(self): return check_hash(self.path / f"{self.month:02d}.params.nc")[0] def register(self, dataset: xr.Dataset): - dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc") + from .._base import _get_xr_engine + + engine = _get_xr_engine() + logger.info(f"register: Saving monthly file with engine={engine}") + dataset.to_netcdf(self.path / f"{self.month:02d}.params.nc", engine=engine) with open(self.path / f"{self.month:02d}.params.nc", "rb") as f: self._hashes[f"{self.month:02d}.params.nc"] = hashlib.sha256( f.read() diff --git a/src/geodata/model/wind/_base.py b/src/geodata/model/wind/_base.py index 244f81e3..370d154a 100644 --- a/src/geodata/model/wind/_base.py +++ b/src/geodata/model/wind/_base.py @@ -40,6 +40,7 @@ """ import xarray as xr +from typing import Any, cast from ...resource import get_windturbineconfig from .._base import BaseModel @@ -107,7 +108,7 @@ def _estimate_power( ys: slice | None = None, years: slice | None = None, months: slice | None = None, - ) -> None: + ) -> xr.DataArray: """Estimate wind speed at the given locations and times. Args: @@ -135,7 +136,7 @@ def _estimate_power( turbineconf["V"], turbineconf["POW"], bounds_error=False, - fill_value="extrapolate", + fill_value=cast(Any, "extrapolate"), ) # Calculate the power output @@ -147,4 +148,4 @@ def _estimate_power( output_dtypes=[float], ) - return xr.Dataset({"cf": power / turbineconf["P"]}) + return (power / turbineconf["P"]).rename("cf") diff --git a/src/geodata/model/wind/interpolate.py b/src/geodata/model/wind/interpolate.py index eba93d19..9b8e0c26 100644 --- a/src/geodata/model/wind/interpolate.py +++ b/src/geodata/model/wind/interpolate.py @@ -15,6 +15,7 @@ import logging from typing import Hashable +from typing import cast import numpy as np import scipy.interpolate as sinterp @@ -51,9 +52,38 @@ def _memoryview_safe(x: np.ndarray) -> np.ndarray: return x -def _make_interp_coeff(*args, **kwargs): - """Dummy function to handle interpolation coefficients.""" - return sinterp.make_interp_spline(*args, **kwargs).c +def _make_interp_coeff(x, y, k, t, check_finite=False): + """Compute interpolation coefficients for a single block. + + Args: + x: 1D array of x coordinates (must be C-contiguous) + y: Data array to interpolate + k: Spline degree + t: Knot vector + check_finite: Whether to check for finite values + + Returns: + Spline coefficients + """ + logger.debug(f"[_make_interp_coeff] Called with x type: {type(x)}, x shape: {np.asarray(x).shape if hasattr(x, 'shape') else 'no shape'}, " + f"x dtype: {np.asarray(x).dtype if hasattr(x, 'dtype') else type(x)}, " + f"y type: {type(y)}, y shape: {np.asarray(y).shape if hasattr(y, 'shape') else 'no shape'}, " + f"k: {k}, t type: {type(t)}, t shape: {np.asarray(t).shape if hasattr(t, 'shape') else 'no shape'}") + + # Ensure x is C-contiguous and writable + x = _memoryview_safe(np.asarray(x, dtype=float)) + logger.debug(f"[_make_interp_coeff] After _memoryview_safe: x shape: {x.shape}, x dtype: {x.dtype}, x.flags.c_contiguous: {x.flags.c_contiguous}") + + try: + result = sinterp.make_interp_spline(x, y, k=k, t=t, check_finite=check_finite).c + logger.debug(f"[_make_interp_coeff] Successfully computed coefficients, shape: {result.shape}") + return result + except Exception as e: + logger.error(f"[_make_interp_coeff] ERROR in make_interp_spline: {type(e).__name__}: {e}") + logger.error(f"[_make_interp_coeff] x details: shape={x.shape}, dtype={x.dtype}, c_contiguous={x.flags.c_contiguous}") + logger.error(f"[_make_interp_coeff] y details: type={type(y)}, shape={np.asarray(y).shape if hasattr(y, 'shape') else 'N/A'}") + logger.error(f"[_make_interp_coeff] t details: type={type(t)}, shape={np.asarray(t).shape if hasattr(t, 'shape') else 'N/A'}") + raise def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: @@ -69,45 +99,86 @@ def _splrep(a: xr.DataArray, dim: Hashable, k: int = 3) -> xr.Dataset: xr.Dataset: Dataset containing spline parameters. """ + logger.debug(f"[_splrep] Starting with dim={dim}, k={k}, a shape: {a.shape}, a dims: {a.dims}") + # Make sure that dim is on axis 0 a = a.transpose(dim, ...) x: np.ndarray = a.coords[dim].values + logger.debug(f"[_splrep] After transpose: a shape: {a.shape}, x shape: {x.shape}, x dtype: {x.dtype}") if x.dtype.kind == "M": # Same treatment will be applied to x_new. # Allow x_new.dtype==M8[D] and x.dtype==M8[ns], or vice versa x = x.astype("M8[ns]").astype(float) + logger.debug(f"[_splrep] Converted datetime x to float, new dtype: {x.dtype}") + + # Ensure x is C-contiguous and properly typed + x = _memoryview_safe(np.asarray(x, dtype=float)) + logger.debug(f"[_splrep] After _memoryview_safe: x shape: {x.shape}, x dtype: {x.dtype}, x.flags.c_contiguous: {x.flags.c_contiguous}") t = sinterp._bsplines._not_a_knot(x, k=k) + logger.debug(f"[_splrep] Computed knots t, shape: {t.shape}, dtype: {t.dtype}") if isinstance(a.data, array_type("dask")): from dask.array import map_blocks - from dask.diagnostics import ProgressBar + from dask.diagnostics.progress import ProgressBar - logger.debug("Computing interpolation coefficients using Dask.") + logger.debug(f"[_splrep] Data is dask array, chunks: {a.data.chunks}, shape: {a.data.shape}") if len(a.data.chunks[0]) > 1: + logger.debug(f"[_splrep] Rechunking dimension {dim} to -1 (was: {a.data.chunks[0]})") a = a.chunk({dim: -1}) + logger.debug(f"[_splrep] After rechunking, chunks: {a.data.chunks}") pbar = ProgressBar() if logger.level <= logging.INFO: pbar.register() - c = map_blocks( - _make_interp_coeff, - x, - a.data, - k=k, - t=t, - check_finite=False, - dtype=float, - ) + # Create a wrapper function that captures x and t as closures + # This ensures they're passed correctly to each block + def _block_interp_coeff(y_block, x=x, k=k, t=t, check_finite=False): + y_block = np.asarray(y_block) + logger.debug(f"[_block_interp_coeff] Called with y_block type: {type(y_block)}, y_block shape: {y_block.shape}, " + f"x type: {type(x)}, x shape: {x.shape if hasattr(x, 'shape') else 'no shape'}, " + f"t type: {type(t)}, t shape: {t.shape if hasattr(t, 'shape') else 'no shape'}") + + # Handle empty blocks - return empty array with correct shape + if y_block.size == 0 or any(s == 0 for s in y_block.shape): + logger.debug(f"[_block_interp_coeff] Empty block detected, returning empty array with shape: {y_block.shape}") + # Return empty array with same shape as input (coefficients have same shape as input) + return np.empty_like(y_block, dtype=float) + + try: + result = _make_interp_coeff(x, y_block, k=k, t=t, check_finite=check_finite) + logger.debug(f"[_block_interp_coeff] Successfully computed, result shape: {result.shape}") + return result + except Exception as e: + logger.error(f"[_block_interp_coeff] ERROR: {type(e).__name__}: {e}") + raise + + logger.debug(f"[_splrep] Calling map_blocks with a.data shape: {a.data.shape}, chunks: {a.data.chunks}") + logger.debug(f"[_splrep] x closure value: shape={x.shape}, dtype={x.dtype}, c_contiguous={x.flags.c_contiguous}") + logger.debug(f"[_splrep] t closure value: shape={t.shape}, dtype={t.dtype}") + + try: + c = map_blocks( + _block_interp_coeff, + a.data, + dtype=float, + drop_axis=[], + ) + logger.debug(f"[_splrep] map_blocks returned, c type: {type(c)}, c shape: {c.shape if hasattr(c, 'shape') else 'N/A'}") + except Exception as e: + logger.error(f"[_splrep] ERROR in map_blocks: {type(e).__name__}: {e}") + raise if logger.level <= logging.INFO: pbar.unregister() else: + logger.debug(f"[_splrep] Data is numpy array (not dask), shape: {a.data.shape}, dtype: {a.data.dtype}") c = _make_interp_coeff(x, a.data, k=k, t=t, check_finite=False) + logger.debug(f"[_splrep] Computed coefficients (numpy), shape: {c.shape}") return xr.Dataset( data_vars={ @@ -126,8 +197,8 @@ def _splev_ker(c: np.ndarray, t: np.ndarray, k: int, height: np.ndarray) -> np.n return np.atleast_1d(sinterp.splev(height, (t, c, k))) -def _splev(da: xr.DataArray, height: float) -> xr.DataArray: - height = np.atleast_1d(height) +def _splev(da: xr.Dataset, height: float) -> xr.DataArray: + height_arr = np.atleast_1d(height) return xr.apply_ufunc( _splev_ker, da["c"], @@ -136,7 +207,7 @@ def _splev(da: xr.DataArray, height: float) -> xr.DataArray: vectorize=True, dask="parallelized", output_dtypes=[da["c"].dtype], - kwargs={"t": da.attrs["t"], "k": da.attrs["k"], "height": height}, + kwargs={"t": da.attrs["t"], "k": da.attrs["k"], "height": height_arr}, ) @@ -146,6 +217,11 @@ class WindInterpolationModel(WindBaseModel): This model uses the ERA5 3D dataset to estimate wind speed at a given height using spline interpolation. + For ``estimate(..., xs=..., ys=...)``, each spatial slice may use either bound order + (``slice(low, high)`` or ``slice(high, low)``); ``BaseModel.estimate`` normalizes + slices to the coordinate monotonic direction before ``xarray.Dataset.sel``, including + for descending coordinates (e.g. latitude). + Example: >>> from geodata import Dataset @@ -156,11 +232,11 @@ class WindInterpolationModel(WindBaseModel): >>> model.estimate(height=12, xs=slice(1, 2), ys=slice(1, 2), years=slice(2010, 2010), months=slice(1, 2)) """ - SUPPORTED_WEATHER_DATA_CONFIGS = {"wind_3d_hourly"} + SUPPORTED_WEATHER_DATA_CONFIGS = ("wind_3d_hourly", "wind_3d_hourly_test") def _prepare_dataset( self, - ds: xr.Dataset, + source: xr.Dataset, half_precision: bool = True, ) -> xr.Dataset: """Compute wind speed using the ERA5 3D dataset. @@ -174,28 +250,48 @@ def _prepare_dataset( """ assert ( - "model_level" in ds.coords + "model_level" in source.coords ), "Dataset does not contain model levels. Please double-check the dataset." - ds.coords["model_level"] = np.array( - [LEVEL_TO_HEIGHT[int(level)] for level in ds["model_level"].values] + source.coords["model_level"] = np.array( + [LEVEL_TO_HEIGHT[int(level)] for level in source["model_level"].values] ) - ds = ( - ds.rename({"model_level": "height"}) + source = ( + source.rename({"model_level": "height"}) .transpose("height", ...) .sortby("height") ) - logger.debug("Shape of heights: %s", ds["height"].shape) - speeds = (ds["u"] ** 2 + ds["v"] ** 2) ** 0.5 + logger.debug("Shape of heights: %s", source["height"].shape) + speeds = (source["u"] ** 2 + source["v"] ** 2) ** 0.5 + logger.debug(f"[_prepare_dataset] Computed speeds, shape: {speeds.shape}, dims: {speeds.dims}, " + f"is dask: {isinstance(speeds.data, array_type('dask'))}") + logger.info(f"[_prepare_dataset] Calling _splrep with speeds shape: {speeds.shape}") params = _splrep(speeds, "height") + logger.info(f"[_prepare_dataset] _splrep returned params, type: {type(params)}, data_vars: {list(params.data_vars.keys())}") if half_precision: params = params.astype(np.float32) return params - def _estimate_dataset(self, params: xr.Dataset, height: float) -> xr.Dataset: + def _estimate_dataset(self, params: xr.Dataset, **kwargs) -> xr.DataArray: + height = float(kwargs["height"]) params = params.transpose("height", ...) - params = rechunk_dataset(params, force_full_chunk_dims=["height"]) - return _splev(params, height) + params = cast( + xr.Dataset, + rechunk_dataset(params, force_full_chunk_dims=["height"]), + ) + result = _splev(params, height) + + # Some upstream ERA5 pipelines historically use `valid_time` for the time-like + # coordinate. Normalize to `time` so we can enforce consistent dims. + if "valid_time" in result.dims or "valid_time" in result.coords: + result = result.rename({"valid_time": "time"}) + + # Standardize output dimension order across wind models: + # `("time", "x", "y")` (wind interpolation should match pvlib). + desired_order = ("time", "x", "y") + if all(d in result.dims for d in desired_order): + result = result.transpose(*desired_order) + return result diff --git a/src/geodata/plot.py b/src/geodata/plot.py index f6e47872..f12c1fdf 100644 --- a/src/geodata/plot.py +++ b/src/geodata/plot.py @@ -22,7 +22,7 @@ import matplotlib.pyplot as plt import xarray as xr -from .cutout import ds_reformat_index +from .mask.spatial import ds_reformat_index from .mask import show # noqa: F401 plt.rcParams["animation.html"] = "jshtml" diff --git a/tests/fixtures/era5/wind_3d_hourly_test/2016/01/01.nc b/tests/fixtures/era5/wind_3d_hourly_test/2016/01/01.nc new file mode 100644 index 00000000..1fa4d084 Binary files /dev/null and b/tests/fixtures/era5/wind_3d_hourly_test/2016/01/01.nc differ diff --git a/tests/fixtures/era5/wind_solar_hourly_test/2016/01.nc b/tests/fixtures/era5/wind_solar_hourly_test/2016/01.nc new file mode 100644 index 00000000..901d16ed Binary files /dev/null and b/tests/fixtures/era5/wind_solar_hourly_test/2016/01.nc differ diff --git a/tests/pr/mask/test_mask_legacy_error_paths.py b/tests/pr/mask/test_mask_legacy_error_paths.py new file mode 100644 index 00000000..7770be6c --- /dev/null +++ b/tests/pr/mask/test_mask_legacy_error_paths.py @@ -0,0 +1,110 @@ +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pytest +import xarray as xr +from rasterio.transform import from_bounds + +from geodata.cutout import Cutout +from geodata.mask import Mask, save_raster + + +def _build_minimal_cutout() -> Cutout: + cutout = Cutout.__new__(Cutout) + cutout.name = "legacy-error-cutout" + cutout.meta = xr.Dataset( + coords={ + "x": np.array([100.0, 100.25, 100.5]), + "y": np.array([30.5, 30.25, 30.0]), + "year": [2016], + "month": [1], + } + ) + cutout.merged_mask = None + cutout.shape_mask = None + cutout.area = None + cutout.prepared = True + cutout.empty = False + cutout.cutout_dir = Path(".") + return cutout + + +def _sample_dataset() -> xr.Dataset: + t = np.array(["2016-01-01T00:00:00"], dtype="datetime64[ns]") + y = np.array([30.5, 30.25, 30.0]) + x = np.array([100.0, 100.25, 100.5]) + data = np.arange(len(t) * len(y) * len(x), dtype=np.float32).reshape( + len(t), len(y), len(x) + ) + return xr.Dataset( + {"signal": (("time", "y", "x"), data)}, + coords={"time": t, "y": y, "x": x}, + ) + + +def _create_saved_empty_mask(mask_dir: Path, name: str) -> None: + # Create a mask object that is saved but has no merged/shape masks. + mask = Mask(name=name, mask_dir=str(mask_dir)) + mask.save_mask() + + +def _create_unsaved_mask_with_layer(mask_dir: Path, name: str) -> Mask: + west, south, east, north = 100.0, 30.0, 100.75, 30.75 + arr = np.ones((3, 3), dtype=np.uint8) + transform = from_bounds(west, south, east, north, arr.shape[1], arr.shape[0]) + layer_path = mask_dir / f"{name}.tif" + save_raster(arr, transform, str(layer_path)) + mask = Mask(name=name, mask_dir=str(mask_dir)) + mask.add_layer(str(layer_path), layer_name="base") + return mask + + +def test_mask_raises_without_added_masks(): + cutout = _build_minimal_cutout() + ds = _sample_dataset() + + with pytest.raises(ValueError, match="No mask found in cutout"): + cutout.mask(ds) + + +def test_mask_raises_when_true_area_requested_without_area(): + cutout = _build_minimal_cutout() + ds = _sample_dataset() + cutout.merged_mask = xr.DataArray( + np.ones((1, 3, 3), dtype=np.float32), + dims=("band", "lat", "lon"), + coords={ + "band": [1], + "lat": ds["y"].values, + "lon": ds["x"].values, + }, + ) + + with pytest.raises(ValueError, match="No area data found"): + cutout.mask(ds, true_area=True) + + +def test_add_mask_raises_for_saved_mask_without_merged_or_shape(tmp_path, monkeypatch): + cutout = _build_minimal_cutout() + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + name = "empty_saved_mask" + _create_saved_empty_mask(mask_dir, name) + + monkeypatch.setattr("geodata.cutout.config.MASK_DIR", str(mask_dir)) + + with pytest.raises(ValueError, match=f"No mask found in {name}"): + cutout.add_mask(name) + + +def test_mask_load_xarray_raises_when_unsaved(tmp_path): + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask = _create_unsaved_mask_with_layer(mask_dir, name="unsaved_mask") + + with pytest.raises(ValueError, match="has not been saved"): + mask.load_merged_xr() + + with pytest.raises(ValueError, match="has not been saved"): + _ = mask.load_shape_xr(names=cast(Any, [])) diff --git a/tests/pr/mask/test_mask_legacy_workflow.py b/tests/pr/mask/test_mask_legacy_workflow.py new file mode 100644 index 00000000..6d75a43e --- /dev/null +++ b/tests/pr/mask/test_mask_legacy_workflow.py @@ -0,0 +1,173 @@ +import uuid +from pathlib import Path +from typing import Any, cast + +import numpy as np +import shapely.geometry +import xarray as xr +from rasterio.transform import from_bounds + +from geodata.cutout import Cutout, calc_grid_area, coarsen, ds_reformat_index +from geodata.datasets import load_dataset +from geodata.mask import Mask, save_raster + + +def _build_cutout(tmp_path: Path) -> Cutout: + dataset_cls = load_dataset("wind_solar_hourly_test") + dataset = dataset_cls(years=slice(2016, 2016), months=slice(1, 1), testing=True) + assert dataset.downloaded, "Fixture NetCDF should be present" + + with xr.open_dataset(dataset.catalog[0].path, engine="h5netcdf") as opened: + if "x" in opened.coords and "y" in opened.coords: + xvals = opened["x"].values + yvals = opened["y"].values + else: + xvals = opened["longitude"].values + yvals = opened["latitude"].values + + # Use a lightweight Cutout instance that still exercises legacy methods + # (add_mask, add_grid_area, mask) without invoking dataset preparation. + cutout = Cutout.__new__(Cutout) + cutout.name = f"legacy-mask-test-{uuid.uuid4().hex[:8]}" + cutout.meta = xr.Dataset( + coords={ + "x": xvals, + "y": yvals, + "year": [2016], + "month": [1], + } + ) + cutout.merged_mask = None + cutout.shape_mask = None + cutout.area = None + cutout.prepared = True + cutout.empty = False + cutout.cutout_dir = tmp_path / "cutouts" + return cutout + + +def _create_and_save_mask(cutout: Cutout, mask_dir: Path, name: str = "legacy_test_mask") -> None: + assert cutout.meta is not None + xr_meta = ds_reformat_index(cast(Any, cutout.meta)) + lon = xr_meta["lon"].values + lat = xr_meta["lat"].values + + lon_step = float(np.abs(lon[1] - lon[0])) + lat_step = float(np.abs(lat[1] - lat[0])) + west = float(lon.min() - lon_step / 2) + east = float(lon.max() + lon_step / 2) + south = float(lat.min() - lat_step / 2) + north = float(lat.max() + lat_step / 2) + + nlon_hi = len(lon) * 2 + nlat_hi = len(lat) * 2 + transform = from_bounds(west, south, east, north, nlon_hi, nlat_hi) + + raster = np.zeros((nlat_hi, nlon_hi), dtype=np.uint8) + # Non-trivial pattern so coarsening does real work. + raster[nlat_hi // 4 : 3 * nlat_hi // 4, nlon_hi // 6 : 5 * nlon_hi // 6] = 1 + + layer_path = mask_dir / "source_layer.tif" + save_raster(raster, transform, str(layer_path)) + + mask = Mask(name=name, mask_dir=str(mask_dir)) + mask.add_layer(str(layer_path), layer_name="source") + mask.merge_layer(show_raster=False) + + centroid_lon = float(np.mean([west, east])) + centroid_lat = float(np.mean([south, north])) + shape = shapely.geometry.box( + west, + south, + centroid_lon, + centroid_lat, + ) + mask.extract_shapes({"region_a": shape}, show_raster=False) + mask.save_mask() + + +def test_legacy_mask_workflow_contract_offline(tmp_path, monkeypatch): + cutout = _build_cutout(tmp_path) + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask_name = "legacy_test_mask" + _create_and_save_mask(cutout, mask_dir, name=mask_name) + + monkeypatch.setattr("geodata.cutout.config.MASK_DIR", str(mask_dir)) + cutout.add_mask(mask_name) + cutout.add_grid_area() + + time = np.array(["2016-01-01T00:00:00", "2016-01-01T01:00:00"], dtype="datetime64[ns]") + y = cutout.coords["y"].values + x = cutout.coords["x"].values + payload = np.arange(len(time) * len(y) * len(x), dtype=np.float32).reshape( + len(time), len(y), len(x) + ) + ds = xr.Dataset( + {"signal": (("time", "y", "x"), payload)}, + coords={"time": time, "y": y, "x": x}, + ) + + masked = cutout.mask(ds) + + assert set(masked.keys()) == {"merged_mask", "region_a"} + merged = masked["merged_mask"] + assert isinstance(merged, xr.Dataset) + assert {"signal", "mask", "area"}.issubset(set(merged.data_vars)) + assert tuple(merged["signal"].dims) == ("time", "lat", "lon") + assert tuple(merged["mask"].dims) == ("lat", "lon") + assert tuple(merged["area"].dims) == ("lat", "lon") + + +def test_legacy_add_mask_coarsen_parity_offline(tmp_path, monkeypatch): + cutout = _build_cutout(tmp_path) + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask_name = "legacy_test_mask" + _create_and_save_mask(cutout, mask_dir, name=mask_name) + + monkeypatch.setattr("geodata.cutout.config.MASK_DIR", str(mask_dir)) + cutout.add_mask(mask_name, shape_mask=False) + + mask = Mask.from_name(mask_name, mask_dir=str(mask_dir)) + assert cutout.meta is not None + expected = coarsen( + cast(Any, mask.load_merged_xr()), + cast(Any, ds_reformat_index(cast(Any, cutout.meta))), + ) + + assert cutout.merged_mask is not None + np.testing.assert_allclose(cutout.merged_mask.values, expected.values) + assert cutout.merged_mask.shape == expected.shape + + +def test_legacy_add_grid_area_sanity_offline(tmp_path): + cutout = _build_cutout(tmp_path) + cutout.add_grid_area() + + assert cutout.area is not None + area = cutout.area["area"].values + assert np.all(np.isfinite(area)) + assert np.all(area > 0) + + # Area should be constant across longitude for a given latitude row. + row_std = area.std(axis=1) + assert np.allclose(row_std, 0.0, atol=1e-6) + + assert cutout.meta is not None + xr_ds = ds_reformat_index(cast(Any, cutout.meta)) + lat = xr_ds.lat.values + lon = xr_ds.lon.values + lat_diff = float(np.abs(lat[1] - lat[0])) + expected_first_row = np.round( + calc_grid_area( + [ + (lon[0], lat[0] + lat_diff / 2), + (lon[0], lat[0] - lat_diff / 2), + (lon[1], lat[0] - lat_diff / 2), + (lon[1], lat[0] + lat_diff / 2), + ] + ), + 2, + ) + assert np.isclose(area[0, 0], expected_first_row) diff --git a/tests/pr/mask/test_mask_merge_inmemory.py b/tests/pr/mask/test_mask_merge_inmemory.py new file mode 100644 index 00000000..0a87346d --- /dev/null +++ b/tests/pr/mask/test_mask_merge_inmemory.py @@ -0,0 +1,93 @@ +"""Regression tests for merge_layer with in-memory (/vsimem) layers.""" + +from pathlib import Path + +import numpy as np +from rasterio.transform import from_bounds + +from geodata.mask import Mask, save_raster + + +def _write_layer(path: Path, west: float, south: float, east: float, north: float, pattern: str): + nlon, nlat = 8, 6 + transform = from_bounds(west, south, east, north, nlon, nlat) + arr = np.zeros((nlat, nlon), dtype=np.uint8) + if pattern == "left": + arr[:, : nlon // 2] = 1 + elif pattern == "right": + arr[:, nlon // 2 :] = 1 + else: + arr[nlat // 4 : 3 * nlat // 4, nlon // 4 : 3 * nlon // 4] = 1 + save_raster(arr, transform, str(path)) + return transform + + +def _mask_with_filtered_layers(tmp_path: Path, *, overlap: bool = False) -> Mask: + west, south, east, north = 100.0, 30.0, 101.0, 31.0 + layer_a = tmp_path / "layer_a.tif" + layer_b = tmp_path / "layer_b.tif" + if overlap: + _write_layer(layer_a, west, south, east, north, "center") + _write_layer(layer_b, west, south, east, north, "center") + else: + _write_layer(layer_a, west, south, east, north, "left") + _write_layer(layer_b, west, south, east, north, "right") + + mask = Mask("inmemory_merge_test", mask_dir=str(tmp_path / "masks")) + mask.add_layer(str(layer_a), layer_name="a") + mask.add_layer(str(layer_b), layer_name="b") + mask.filter_layer("a", min_bound=0.5, binarize=True, dest_layer_name="a") + mask.filter_layer("b", min_bound=0.5, binarize=True, dest_layer_name="b") + return mask + + +def test_filtered_layers_are_vsimem_backed(tmp_path): + mask = _mask_with_filtered_layers(tmp_path) + for ds in mask.layers.values(): + assert ds.name.startswith("/vsimem"), ds.name + ds.read(1) + + +def test_merge_and_after_filter_layer(tmp_path): + mask = _mask_with_filtered_layers(tmp_path) + merged = mask.merge_layer( + method="and", + layers=["a", "b"], + reference_layer="a", + show_raster=False, + ) + assert not merged.closed + data = merged.read(1) + assert data.shape == (6, 8) + assert mask.merged_mask is not None + assert not mask.saved + + +def test_merge_sum_after_filter_layer(tmp_path): + mask = _mask_with_filtered_layers(tmp_path) + merged = mask.merge_layer( + method="sum", + layers=["a", "b"], + weights={"a": 1.0, "b": 2.0}, + reference_layer="a", + show_raster=False, + attribute_save=False, + ) + assert not merged.closed + data = merged.read(1) + assert np.any(data > 0) + + +def test_merge_and_trim_after_filter(tmp_path): + mask = _mask_with_filtered_layers(tmp_path, overlap=True) + merged = mask.merge_layer( + method="and", + layers=["a", "b"], + reference_layer="a", + trim=True, + show_raster=False, + ) + data = merged.read(1) + assert data.shape[0] <= 6 + assert data.shape[1] <= 8 + assert np.any(data != 0) diff --git a/tests/pr/mask/test_mask_spatial_helpers.py b/tests/pr/mask/test_mask_spatial_helpers.py new file mode 100644 index 00000000..af8ba604 --- /dev/null +++ b/tests/pr/mask/test_mask_spatial_helpers.py @@ -0,0 +1,49 @@ +import numpy as np +import xarray as xr + +from geodata.mask.spatial import calc_grid_area, coarsen, ds_reformat_index + + +def test_ds_reformat_index_renames_and_sorts_xy(): + x = np.array([101.0, 100.5, 100.0]) + y = np.array([30.0, 30.5, 31.0]) + arr = np.arange(9, dtype=np.float32).reshape(3, 3) + da = xr.DataArray(arr, dims=("y", "x"), coords={"x": x, "y": y}, name="signal") + + out = ds_reformat_index(da) + assert out.dims == ("lat", "lon") + assert np.all(np.diff(out["lat"].values) >= 0) + assert np.all(np.diff(out["lon"].values) >= 0) + + +def test_coarsen_mean_on_aligned_grid(): + lat_hi = np.array([0.0, 0.25, 0.5, 0.75]) + lon_hi = np.array([10.0, 10.25, 10.5, 10.75]) + hi = xr.DataArray( + np.arange(16, dtype=np.float32).reshape(4, 4), + dims=("lat", "lon"), + coords={"lat": lat_hi, "lon": lon_hi}, + name="mask", + ) + + lat_lo = np.array([0.125, 0.625]) + lon_lo = np.array([10.125, 10.625]) + lo = xr.Dataset(coords={"lat": lat_lo, "lon": lon_lo}) + + out = coarsen(hi, lo, func="mean") + # Freeze current legacy coarsen behavior. + expected = np.array([[7.5, 9.0], [13.5, 15.0]], dtype=np.float32) + np.testing.assert_allclose(out.values, expected, atol=1e-6) + + +def test_calc_grid_area_positive_and_latitude_sensitive(): + # Avoid perfectly symmetric parallels around 0 that can trip AEA constraints. + cell_low_lat = [(0.0, 1.5), (0.0, 0.5), (1.0, 0.5), (1.0, 1.5)] + cell_high_lat = [(0.0, 60.5), (0.0, 59.5), (1.0, 59.5), (1.0, 60.5)] + + area_low_lat = calc_grid_area(cell_low_lat) + area_high_lat = calc_grid_area(cell_high_lat) + + assert area_low_lat > 0 + assert area_high_lat > 0 + assert area_low_lat > area_high_lat diff --git a/tests/pr/mask/test_xarray_mask.py b/tests/pr/mask/test_xarray_mask.py new file mode 100644 index 00000000..02ab2e33 --- /dev/null +++ b/tests/pr/mask/test_xarray_mask.py @@ -0,0 +1,136 @@ +from pathlib import Path +from typing import Any, cast + +import numpy as np +import shapely.geometry +import xarray as xr +import rasterio as ras +from rasterio.transform import from_bounds + +from geodata import Mask, XarrayMask +from geodata.cutout import Cutout, ds_reformat_index + + +def _build_minimal_cutout() -> Cutout: + cutout = Cutout.__new__(Cutout) + cutout.name = "xarray-mask-test" + cutout.meta = xr.Dataset( + coords={ + "x": np.array([100.0, 100.25, 100.5, 100.75]), + "y": np.array([30.75, 30.5, 30.25, 30.0]), + "year": [2016], + "month": [1], + } + ) + cutout.merged_mask = None + cutout.shape_mask = None + cutout.area = None + cutout.prepared = True + cutout.empty = False + cutout.cutout_dir = Path(".") + return cutout + + +def _create_saved_mask(cutout: Cutout, mask_dir: Path, name: str = "xarray_test_mask") -> None: + assert cutout.meta is not None + xr_meta = ds_reformat_index(cast(Any, cutout.meta)) + lon = xr_meta["lon"].values + lat = xr_meta["lat"].values + + lon_step = float(np.abs(lon[1] - lon[0])) + lat_step = float(np.abs(lat[1] - lat[0])) + west = float(lon.min() - lon_step / 2) + east = float(lon.max() + lon_step / 2) + south = float(lat.min() - lat_step / 2) + north = float(lat.max() + lat_step / 2) + + nlon_hi = len(lon) * 2 + nlat_hi = len(lat) * 2 + transform = from_bounds(west, south, east, north, nlon_hi, nlat_hi) + + arr = np.zeros((nlat_hi, nlon_hi), dtype=np.uint8) + arr[nlat_hi // 4 : 3 * nlat_hi // 4, nlon_hi // 4 : 3 * nlon_hi // 4] = 1 + layer_path = mask_dir / "source.tif" + with ras.open( + str(layer_path), + "w", + driver="GTiff", + height=arr.shape[0], + width=arr.shape[1], + count=1, + dtype=arr.dtype, + compress="lzw", + crs="+proj=latlong", + transform=transform, + ) as dst: + dst.write(arr, 1) + + mask = Mask(name=name, mask_dir=str(mask_dir)) + mask.add_layer(str(layer_path), layer_name="source") + mask.merge_layer(show_raster=False) + shape = shapely.geometry.box(west, south, (west + east) / 2, (south + north) / 2) + mask.extract_shapes({"region_a": shape}, show_raster=False) + mask.save_mask() + + +def _sample_dataset_from_cutout(cutout: Cutout) -> xr.Dataset: + assert cutout.meta is not None + y = cutout.meta["y"].values + x = cutout.meta["x"].values + t = np.array(["2016-01-01T00:00:00", "2016-01-01T01:00:00"], dtype="datetime64[ns]") + vals = np.arange(len(t) * len(y) * len(x), dtype=np.float32).reshape( + len(t), len(y), len(x) + ) + return xr.Dataset({"signal": (("time", "y", "x"), vals)}, coords={"time": t, "y": y, "x": x}) + + +def test_xarraymask_attach_matches_legacy_contract(tmp_path, monkeypatch): + cutout = _build_minimal_cutout() + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask_name = "xarray_test_mask" + _create_saved_mask(cutout, mask_dir, name=mask_name) + + monkeypatch.setattr("geodata.cutout.config.MASK_DIR", str(mask_dir)) + cutout.add_mask(mask_name) + cutout.add_grid_area() + + ds = _sample_dataset_from_cutout(cutout) + legacy = cutout.mask(ds) + + assert cutout.meta is not None + xmask = XarrayMask.from_name(mask_name, grid=cutout.meta, mask_dir=str(mask_dir)) + attached = xmask.attach(ds, include_area=True) + + assert set(attached.keys()) == set(legacy.keys()) + for key in attached: + xr.testing.assert_allclose(attached[key]["mask"], legacy[key]["mask"]) + xr.testing.assert_allclose(attached[key]["area"], legacy[key]["area"]) + xr.testing.assert_allclose(attached[key]["signal"], legacy[key]["signal"]) + + +def test_xarraymask_apply_where_and_multiply(tmp_path): + cutout = _build_minimal_cutout() + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask_name = "xarray_test_mask" + _create_saved_mask(cutout, mask_dir, name=mask_name) + + ds = _sample_dataset_from_cutout(cutout) + assert cutout.meta is not None + xmask = XarrayMask.from_name(mask_name, grid=cutout.meta, mask_dir=str(mask_dir)) + + attached = xmask.attach(ds, include_area=False) + merged_mask = attached["merged_mask"]["mask"] + + where_out = xmask.apply(ds, mode="where", include_area=True)["merged_mask"] + multiply_out = xmask.apply(ds, mode="multiply", include_area=False)["merged_mask"] + + valid = merged_mask > 0 + expected_where = attached["merged_mask"]["signal"].where(valid) + expected_multiply = attached["merged_mask"]["signal"] * valid + + xr.testing.assert_allclose(where_out["signal"], expected_where) + xr.testing.assert_allclose(multiply_out["signal"], expected_multiply) + assert "area" in where_out + assert "area" not in multiply_out diff --git a/tests/pr/test_dataset_comprehensive.py b/tests/pr/test_dataset_comprehensive.py new file mode 100644 index 00000000..9f0ff398 --- /dev/null +++ b/tests/pr/test_dataset_comprehensive.py @@ -0,0 +1,729 @@ +# Copyright 2024 Xiqiang Liu + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Comprehensive examples of different types of tests for dataset classes. + +This file demonstrates various testing patterns for geospatial datasets. +Each test category serves a specific purpose in ensuring dataset quality, +correctness, and reliability. + +Test Categories Explained: +-------------------------- + +1. DOWNLOAD TESTS + - Verify that datasets can be downloaded successfully + - Ensure files are saved to correct locations + - Critical for ensuring the basic data acquisition pipeline works + +2. CATALOG TESTS + - Validate that the catalog (list of files to download) is correctly generated + - Test different frequencies (monthly, daily, hourly) + - Verify testing mode limits downloads appropriately + - Important for understanding what files will be downloaded before actually downloading + +3. DATA INTEGRITY TESTS + - Check file checksums/hashes to detect corruption + - Verify downloaded files can be opened and read + - Ensure data hasn't been corrupted during download or storage + - Critical for data quality assurance + +4. COORDINATE & BOUNDS TESTS + - Validate coordinate system transformations (lat/lon to x/y) + - Test bounding box filtering works correctly + - Verify coordinate ranges are within expected limits + - Important for spatial data correctness + +5. METADATA TESTS + - Verify dataset properties (projection, lat_direction, frequency) + - Test that required attributes are present + - Validate metadata is consistent across dataset types + - Important for understanding dataset characteristics + +6. DATA STRUCTURE TESTS + - Verify downloaded datasets have expected variables + - Check coordinate dimensions match expectations + - Validate data types and value ranges + - Critical for ensuring data usability + +7. POSTPROCESSING TESTS + - Test that dataset postprocessing functions correctly + - Verify coordinate renaming (lat/lon -> x/y) + - Check data transformations are applied correctly + - Important for ensuring data is in the expected format + +8. MULTI-DATASET TESTS + - Compare outputs from different datasets for consistency + - Test interoperability between different dataset types + - Important for ensuring datasets can be used together +""" + +import logging +import os +from typing import Optional + +import xarray as xr + +from geodata.datasets import load_dataset + +logging.basicConfig(level=logging.INFO) + +# PRs should run with zero CDS calls. Enable integration download tests only via: +# GEODATA_RUN_CDS_TESTS=1 +RUN_CDS_TESTS = os.getenv("GEODATA_RUN_CDS_TESTS") == "1" + + +# ============================================================================ +# TEST CONFIGURATION HELPERS +# ============================================================================ + +def get_data_configs() -> list[str]: + """Get list of dataset configurations to test.""" + # Default to offline fixtures for PR safety. + return ["wind_3d_hourly"] if RUN_CDS_TESTS else ["wind_3d_hourly_test"] + + +def get_bounds() -> list[list[float]]: + """Get list of bounding boxes to test (lon_min, lat_min, lon_max, lat_max).""" + # Bounds are only meaningful for real downloads. Fixture files are not + # regenerated per-bounds and therefore shouldn't be validated against bounds. + return [[50, 0, 48, 3]] if RUN_CDS_TESTS else [None] # type: ignore[list-item] + + +def get_years() -> list[slice]: + """Get list of year ranges to test.""" + return [slice(2005, 2005)] if RUN_CDS_TESTS else [slice(2016, 2016)] + + +def get_months() -> list[slice]: + """Get list of month ranges to test.""" + return [slice(1, 2)] if RUN_CDS_TESTS else [slice(1, 1)] + + +def get_dataset( + data_config: str, + bound: Optional[list[float]], + year: slice, + month: slice, + testing: bool = True, +): + """Helper function to create and optionally download a dataset.""" + dataset_cls = load_dataset(data_config) + dataset = dataset_cls( + years=year, months=month, bounds=bound, testing=testing + ) + if not dataset.downloaded: + if RUN_CDS_TESTS: + dataset.download() + else: + raise AssertionError( + f"Dataset {data_config} is not downloaded, but CDS tests are disabled. " + "Use fixture configs or set GEODATA_RUN_CDS_TESTS=1." + ) + return dataset + + +# ============================================================================ +# 1. DOWNLOAD TESTS +# ============================================================================ + +def test_download(): + """ + Test Category 1: Download Tests + + WHY: Ensures the basic data acquisition pipeline works correctly. + Downloads are expensive (time, bandwidth, storage), so we need to verify + they work before running longer tests. This is the foundation for all + other data-dependent tests. + """ + if not RUN_CDS_TESTS: + # This test verifies CDS download pipeline. Keep it opt-in. + return + + configs = get_data_configs() + years = get_years() + months = get_months() + bounds = get_bounds() + + for config, year, month, bound in zip(configs, years, months, bounds): + dataset = get_dataset(config, bound, year, month) + assert dataset.downloaded, f"Dataset {config} should be downloaded" + + +# ============================================================================ +# 2. CATALOG TESTS +# ============================================================================ + +def test_catalog_generation(): + """ + Test Category 2: Catalog Generation Tests + + WHY: The catalog determines which files need to be downloaded. Incorrect + catalog generation means missing data or unnecessary downloads. Testing + this ensures we know exactly what will be downloaded before we download it. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + + # Test monthly catalog (if applicable) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + testing=True, + ) + catalog = dataset.catalog + + assert len(catalog) > 0, "Catalog should contain at least one file" + + # Verify catalog entries have correct structure + for file in catalog: + assert hasattr(file, "year"), "Catalog entry should have year" + assert hasattr(file, "month"), "Catalog entry should have month" + assert hasattr(file, "path"), "Catalog entry should have path" + assert file.year == (2005 if RUN_CDS_TESTS else 2016), "Year should match" + assert file.month == 1, "Month should match" + + +def test_catalog_testing_mode(): + """ + Test Category 2: Testing Mode Catalog Tests + + WHY: Testing mode should limit downloads to a few days/months to speed up + tests. If this doesn't work correctly, tests become slow and expensive. + """ + if not RUN_CDS_TESTS: + # Fixture datasets have fixed catalogs; testing mode isn't meaningful here. + return + + config = "wind_3d_hourly" + dataset_cls = load_dataset(config) + + # Testing mode should limit to 3 days for daily frequency datasets + dataset_testing = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + testing=True + ) + catalog_testing = dataset_testing.catalog + + # Non-testing mode would download full month + dataset_normal = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + testing=False + ) + catalog_normal = dataset_normal.catalog + + # Testing mode should have fewer files + assert len(catalog_testing) < len(catalog_normal), \ + "Testing mode should limit the number of files" + + +def test_catalog_paths(): + """ + Test Category 2: Catalog Path Tests + + WHY: File paths determine where data is stored. Incorrect paths lead to + data being saved in wrong locations or files overwriting each other. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + testing=True, + ) + + catalog = dataset.catalog + paths = {file.path for file in catalog} + + # All paths should be unique + assert len(paths) == len(catalog), "All catalog paths should be unique" + + # Paths should follow expected structure (year/month/day.nc for daily) + for file in catalog: + path_str = str(file.path) + assert str(file.year) in path_str, "Path should contain year" + assert f"{file.month:02d}" in path_str, "Path should contain month" + if file.day is not None: + assert f"{file.day:02d}.nc" in path_str, "Path should contain day for daily datasets" + + +# ============================================================================ +# 3. DATA INTEGRITY TESTS +# ============================================================================ + +def test_file_integrity(): + """ + Test Category 3: File Integrity Tests + + WHY: Downloaded files can become corrupted during transfer or storage. + Integrity checks catch these issues before they cause problems in analysis. + """ + configs = get_data_configs() + years = get_years() + months = get_months() + bounds = get_bounds() + + for config, year, month, bound in zip(configs, years, months, bounds): + dataset = get_dataset(config, bound, year, month) + + # Check integrity of all files in catalog + for file in dataset.catalog: + assert file.check(), f"File {file.path} should exist" + + # Test integrity check (requires file_hash to be set) + # Note: This would require files to have hashes stored + assert file.check(integrity=False), \ + f"File {file.path} should pass basic integrity check" + + +def test_file_readability(): + """ + Test Category 3: File Readability Tests + + WHY: A file can exist and pass checksum but still be unreadable (wrong format, + corrupted headers, etc.). This ensures we can actually use the downloaded data. + """ + configs = get_data_configs() + years = get_years() + months = get_months() + bounds = get_bounds() + + for config, year, month, bound in zip(configs, years, months, bounds): + dataset = get_dataset(config, bound, year, month) + + for file in dataset.catalog: + if file.path.exists(): + # Should be able to open as xarray dataset + ds = xr.open_dataset(file.path) + assert ds is not None, f"Should be able to open {file.path}" + ds.close() + + +# ============================================================================ +# 4. COORDINATE & BOUNDS TESTS +# ============================================================================ + +def test_bounds_validation(): + """ + Test Category 4: Bounds Validation Tests + + WHY: Bounding boxes filter data spatially. Incorrect bounds can lead to + downloading unnecessary data or missing required data. Also validates that + invalid bounds are rejected early. + """ + if not RUN_CDS_TESTS: + return + + config = "wind_3d_hourly" + dataset_cls = load_dataset(config) + + # Test valid bounds + valid_bounds = [50, 0, 52, 3] # lon_min, lat_min, lon_max, lat_max + dataset = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + bounds=valid_bounds, + testing=True + ) + assert dataset.bounds == valid_bounds, "Valid bounds should be accepted" + + # Note: To test invalid bounds validation, you could add a test that + # verifies ValueError is raised for bounds outside valid ranges. + # Example: bounds with longitude > 180 or < -180 should raise ValueError + + +def test_coordinate_renaming(): + """ + Test Category 4: Coordinate Renaming Tests + + WHY: Datasets use different coordinate names (lat/lon vs x/y). The base + class should standardize these. Incorrect renaming breaks downstream analysis. + """ + configs = get_data_configs() + years = get_years() + months = get_months() + bounds = get_bounds() + + for config, year, month, bound in zip(configs, years, months, bounds): + dataset = get_dataset(config, bound, year, month) + + # Check at least one file to verify coordinate naming + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # After postprocessing, coordinates should be renamed to x, y + # (or lat, lon should be present if add_lon_lat=True) + coords = list(ds.coords.keys()) + + # Should have x and y coordinates (or lat/lon) + has_xy = "x" in coords and "y" in coords + has_latlon = "lat" in coords and "lon" in coords + + assert has_xy or has_latlon, \ + f"Dataset should have x/y or lat/lon coordinates. Found: {coords}" + + ds.close() + break # Only check first file + + +# ============================================================================ +# 5. METADATA TESTS +# ============================================================================ + +def test_dataset_properties(): + """ + Test Category 5: Dataset Properties Tests + + WHY: Dataset properties (projection, lat_direction, frequency) are used + throughout the codebase for processing. Incorrect properties break analysis. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + testing=True, + ) + + # Test required properties exist + assert hasattr(dataset, "projection"), "Dataset should have projection property" + assert hasattr(dataset, "lat_direction"), "Dataset should have lat_direction property" + assert hasattr(dataset, "frequency"), "Dataset should have frequency property" + assert hasattr(dataset, "module"), "Dataset should have module attribute" + assert hasattr(dataset, "weather_config"), "Dataset should have weather_config attribute" + + # Test property types + assert isinstance(dataset.projection, str), "Projection should be a string" + assert isinstance(dataset.lat_direction, bool), "lat_direction should be a boolean" + assert dataset.frequency in ["hourly", "daily", "monthly"], \ + "Frequency should be one of: hourly, daily, monthly" + + +def test_dataset_repr(): + """ + Test Category 5: Dataset Representation Tests + + WHY: The __repr__ method is used for debugging and logging. It should provide + useful information about the dataset state. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + testing=True, + ) + + repr_str = repr(dataset) + + # Should contain key information + assert dataset.weather_config in repr_str, "repr should contain weather_config" + assert ("2005" if RUN_CDS_TESTS else "2016") in repr_str, "repr should contain years" + assert "1" in repr_str, "repr should contain months" + + +# ============================================================================ +# 6. DATA STRUCTURE TESTS +# ============================================================================ + +def test_data_variables(): + """ + Test Category 6: Data Variables Tests + + WHY: Each dataset should contain specific variables. Missing or incorrectly + named variables break downstream analysis that depends on them. + """ + configs = get_data_configs() + years = get_years() + months = get_months() + bounds = get_bounds() + + for config, year, month, bound in zip(configs, years, months, bounds): + dataset = get_dataset(config, bound, year, month) + + # Check if dataset defines expected variables + # Note: Not all datasets have a 'variables' attribute + # This is an example of how to test datasets that do have it + if hasattr(dataset, "variables"): + expected_vars = getattr(dataset, "variables") + + # Verify at least one file contains these variables + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # Variables should exist in dataset + for var in expected_vars: + assert var in ds.data_vars or var in ds.coords, \ + f"Variable {var} should exist in dataset" + + ds.close() + break # Only check first file + + +def test_data_dimensions(): + """ + Test Category 6: Data Dimension Tests + + WHY: Data dimensions determine how data can be processed. For example, + a 3D wind dataset should have a level/height dimension. Missing dimensions + indicate incorrect data structure. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + bounds=get_bounds()[0], + testing=True, + ) + if RUN_CDS_TESTS and not dataset.downloaded: + dataset.download() + + # Check first downloaded file + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # 3D wind data should have multiple dimensions + dims = list(ds.dims.keys()) + + # Should have spatial dimensions + assert "x" in dims or "lon" in dims, "Should have x/lon dimension" + assert "y" in dims or "lat" in dims, "Should have y/lat dimension" + + # 3D data should have a level/height dimension + _ = any(dim in dims for dim in ["level", "height", "lev", "plev"]) + + ds.close() + break # Only check first file + + +def test_data_value_ranges(): + """ + Test Category 6: Data Value Range Tests + + WHY: Data values should be within physically plausible ranges. Out-of-range + values indicate data corruption or processing errors. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + bounds=get_bounds()[0], + testing=True + ) + if RUN_CDS_TESTS and not dataset.downloaded: + dataset.download() + + # Check first downloaded file + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # Check that data values are finite (not NaN or Inf) + for var in ds.data_vars: + data = ds[var] + assert data.notnull().any(), \ + f"Variable {var} should have some non-null values" + + # Wind components should be within reasonable range + # (typical wind speeds are -100 to 100 m/s) + var_str = str(var) + if "u" in var_str.lower() or "v" in var_str.lower(): + if data.notnull().any(): + data_min = float(data.min()) + data_max = float(data.max()) + # Allow wide range, but should be finite + assert abs(data_min) < 200, \ + f"Wind component {var} min value {data_min} seems unreasonable" + assert abs(data_max) < 200, \ + f"Wind component {var} max value {data_max} seems unreasonable" + + ds.close() + break # Only check first file + + +# ============================================================================ +# 7. POSTPROCESSING TESTS +# ============================================================================ + +def test_postprocessing_applied(): + """ + Test Category 7: Postprocessing Application Tests + + WHY: Postprocessing (coordinate renaming, data transformations) must be + applied consistently. If postprocessing fails silently, downstream code + expecting transformed data will fail. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + bounds=get_bounds()[0], + testing=True + ) + if RUN_CDS_TESTS and not dataset.downloaded: + dataset.download() + + # Check that postprocessed files have correct structure + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # Postprocessing should rename coordinates + # Check that we have standardized coordinate names + coords = list(ds.coords.keys()) + assert "x" in coords or "lon" in coords, \ + "Postprocessed data should have x/lon coordinate" + assert "y" in coords or "lat" in coords, \ + "Postprocessed data should have y/lat coordinate" + + ds.close() + break # Only check first file + + +# ============================================================================ +# 8. MULTI-DATASET TESTS (Example - can be expanded) +# ============================================================================ + +def test_datasets_loaded_correctly(): + """ + Test Category 8: Multi-Dataset Loading Tests + + WHY: The dataset registry and loading mechanism must work correctly for + all datasets. If one dataset can't be loaded, it breaks the entire system. + """ + from geodata.datasets import list_datasets, load_dataset + + # Should be able to list all datasets + datasets = list_datasets() + assert len(datasets) > 0, "Should have at least one dataset registered" + + # Should be able to load each dataset class + for dataset_name in datasets: + dataset_cls = load_dataset(dataset_name) + assert dataset_cls is not None, \ + f"Should be able to load dataset class for {dataset_name}" + + +# ============================================================================ +# ADDITIONAL USEFUL TESTS +# ============================================================================ + +def test_testing_mode(): + """ + Additional Test: Testing Mode Behavior + + WHY: Testing mode is crucial for fast CI/CD pipelines. If it doesn't work + correctly, tests become too slow or download too much data. + """ + if not RUN_CDS_TESTS: + # Fixture datasets ignore testing-mode catalog limiting; keep this check opt-in. + return + + config = "wind_3d_hourly" + dataset_cls = load_dataset(config) + + # Testing mode should limit downloads + dataset_testing = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + testing=True + ) + assert dataset_testing.testing is True, "Testing mode should be enabled" + + # Non-testing mode + dataset_normal = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + testing=False + ) + assert dataset_normal.testing is False, "Testing mode should be disabled" + + +def test_storage_path(): + """ + Additional Test: Storage Path Tests + + WHY: Files must be saved to the correct location for proper organization + and retrieval. Wrong paths make it impossible to find downloaded data. + """ + config = "wind_3d_hourly" if RUN_CDS_TESTS else "wind_3d_hourly_test" + dataset_cls = load_dataset(config) + dataset = dataset_cls( + years=slice(2005, 2005) if RUN_CDS_TESTS else slice(2016, 2016), + months=slice(1, 1), + testing=True, + ) + + # Storage root should follow expected pattern + assert dataset.storage_root is not None, "Storage root should be set" + assert "era5" in str(dataset.storage_root), \ + "Storage root should contain module name" + assert dataset.weather_config in str(dataset.storage_root), \ + "Storage root should contain weather_config" + + +def test_bounds_applied(): + """ + Additional Test: Bounds Application Tests + + WHY: When bounds are specified, data should be filtered to those bounds. + Downloading global data when only a region is needed wastes resources. + """ + if not RUN_CDS_TESTS: + return + + config = "wind_3d_hourly" + dataset_cls = load_dataset(config) + + bounds = [50, 0, 52, 3] # Small region + dataset = dataset_cls( + years=slice(2005, 2005), + months=slice(1, 1), + bounds=bounds, + testing=True + ) + dataset.download() + + # Check that downloaded data respects bounds + for file in dataset.catalog: + if file.path.exists(): + ds = xr.open_dataset(file.path) + + # Check coordinate ranges (if coordinates are available) + if "x" in ds.coords: + x_coords = ds.coords["x"].values + lon_min, lon_max = min(x_coords), max(x_coords) + # Data should be within or close to bounds (allowing for grid resolution) + # ERA5 uses 0.25-degree grid, and xr.sel() with slice may include grid points + # that extend beyond requested bounds. We allow up to 2.5 degrees tolerance + # to account for grid alignment and coordinate system conversions. + # Bounds are [lon_min, lat_min, lon_max, lat_max] + tolerance = 2.5 # Degrees tolerance for grid resolution and coordinate conversion + assert lon_min >= bounds[0] - tolerance, \ + f"Longitude min {lon_min} should be >= bounds[0] {bounds[0]} - {tolerance}" + assert lon_max <= bounds[2] + tolerance, \ + f"Longitude max {lon_max} should be <= bounds[2] {bounds[2]} + {tolerance}" + + ds.close() + break # Only check first file + + diff --git a/tests/pr/test_era5_lengthy.py b/tests/pr/test_era5_lengthy.py index 9979ace3..4b06b522 100644 --- a/tests/pr/test_era5_lengthy.py +++ b/tests/pr/test_era5_lengthy.py @@ -16,11 +16,14 @@ """Tests in this file are lengthy due to the nature of the dataset being tested.""" import logging +import os -from geodata.datasets import DatasetType, load_dataset +from geodata.datasets import load_dataset logging.basicConfig(level=logging.INFO) +RUN_CDS_TESTS = os.getenv("GEODATA_RUN_CDS_TESTS") == "1" + # TODO: Test other functionalities with the 3D dataset def get_data_configs() -> list[str]: @@ -41,7 +44,7 @@ def get_months() -> list[slice]: def get_era5(data_config: str, bound: list[int], year: slice, month: slice): dataset_cls = load_dataset(data_config) - dataset: DatasetType = dataset_cls( + dataset = dataset_cls( years=year, months=month, bounds=bound, testing=True ) if not dataset.downloaded: @@ -50,6 +53,11 @@ def get_era5(data_config: str, bound: list[int], year: slice, month: slice): def test_download(): + if not RUN_CDS_TESTS: + # PRs should not require CDS keys / network. Run this test only in + # an opt-in integration job: GEODATA_RUN_CDS_TESTS=1. + return + configs = get_data_configs() years = get_years() months = get_months() diff --git a/tests/pr/test_era5_wind3d.py b/tests/pr/test_era5_wind3d.py new file mode 100644 index 00000000..f27ff9f3 --- /dev/null +++ b/tests/pr/test_era5_wind3d.py @@ -0,0 +1,84 @@ +# Copyright 2025 Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +import xarray as xr +from dask.distributed import Client + +from geodata.datasets import load_dataset +from geodata.datasets._base import BaseDataset +from geodata.logging import logger +from geodata.model.wind import WindInterpolationModel + +logger.setLevel(logging.DEBUG) + + +def _fixture_xy_slices(dataset: BaseDataset) -> tuple[slice, slice]: + """Build ``xs``, ``ys`` slices on the fixture grid (``x``/``y`` or ERA5 ``longitude``/``latitude``).""" + path = dataset.catalog[0].path + with xr.open_dataset(path, engine="h5netcdf") as opened: + if "x" in opened.coords: + xv = opened["x"].values + yv = opened["y"].values + else: + xv = opened["longitude"].values + yv = opened["latitude"].values + xs = slice(float(xv[0]), float(xv[-1])) + ys = slice(float(yv[0]), float(yv[-1])) + return xs, ys + + +def test_wind_interpolation_workflow(): + """Wind interpolation workflow using offline ``wind_3d_hourly_test`` fixtures (no CDS). + + Verifies: + - Fixture dataset is registered and on disk + - Model can be created and prepared + - Capacity factor and wind-speed estimates run on the fixture extent + """ + + years = slice(2016, 2016) + months = slice(1, 1) + + with Client(processes=True, threads_per_worker=1): + ds_cls = load_dataset("wind_3d_hourly_test") + ds = ds_cls(years=years, months=months) + assert ds.downloaded, "Fixture NetCDF should be present" + + xs, ys = _fixture_xy_slices(ds) + + model = WindInterpolationModel(ds) + assert model is not None + model.prepare(force=True) + + turbine_name = "Enercon_E126_7500kW" + + cf_global = model.estimate(turbine=turbine_name) + assert cf_global is not None + assert isinstance(cf_global, (xr.DataArray, xr.Dataset)) + + cf_region = model.estimate(turbine=turbine_name, xs=xs, ys=ys) + assert cf_region is not None + assert isinstance(cf_region, (xr.DataArray, xr.Dataset)) + + speed = model.estimate(height=100.0, xs=xs, ys=ys) + assert speed is not None + assert isinstance(speed, xr.DataArray) + + cf_computed = cf_region.compute() + assert cf_computed is not None + max_cf = cf_computed.max() + assert max_cf is not None diff --git a/tests/pr/test_era5_windsolar.py b/tests/pr/test_era5_windsolar.py new file mode 100644 index 00000000..6c3c00b8 --- /dev/null +++ b/tests/pr/test_era5_windsolar.py @@ -0,0 +1,118 @@ +# Copyright 2025 Keyu Long (UCSD) + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +import xarray as xr +from dask.distributed import Client + +from geodata.datasets import load_dataset +from geodata.datasets._base import BaseDataset +from geodata.logging import logger +from geodata.model.pvlib import Pvlib +from geodata.model.wind import WindInterpolationModel + +logger.setLevel(logging.DEBUG) + + +def _fixture_xy_slices(dataset: BaseDataset) -> tuple[slice, slice]: + """Build ``xs``, ``ys`` slices on the fixture grid (``x``/``y`` or ERA5 ``longitude``/``latitude``).""" + path = dataset.catalog[0].path + with xr.open_dataset(path, engine="h5netcdf") as opened: + if "x" in opened.coords: + xv = opened["x"].values + yv = opened["y"].values + else: + xv = opened["longitude"].values + yv = opened["latitude"].values + xs = slice(float(xv[0]), float(xv[-1])) + ys = slice(float(yv[0]), float(yv[-1])) + return xs, ys + + +def test_wind_solar_workflow(): + """Pvlib + wind interpolation using offline ``*_test`` fixtures (no CDS).""" + + years = slice(2016, 2016) + months = slice(1, 1) + + with Client(processes=True, threads_per_worker=1): + ds_cls = load_dataset("wind_solar_hourly_test") + ds = ds_cls(years=years, months=months) + assert ds.downloaded, "Fixture NetCDF should be present" + + xs, ys = _fixture_xy_slices(ds) + + model = Pvlib(ds) + assert model is not None + + n_mods = 50 + n_strings = 1 + cec_modules = model.retrieve_sam("CECMod") + module = cec_modules["Kaneka_U_SA105"] + inv = model.retrieve_sam("CECInverter")["Fronius_USA__CL_33_3_Delta__208V_"] + model.init_pv_system( + arrays=None, + surface_tilt=35, + surface_azimuth=180, + racking_model="open_rack", + module_parameters=module, + modules_per_string=n_mods, + module_type="glass_polymer", + module="Kaneka_U_SA105", + strings_per_inverter=n_strings, + inverter_parameters=inv, + ) + assert model.pv_system is not None + + model.init_model_config( + clearsky_model="haurwitz", + transposition_model="perez", + solar_position_method="nrel_numpy", + airmass_model="kastenyoung1989", + dc_model="cec", + ac_model="sandia", + aoi_model="physical", + spectral_model="first_solar", + dc_ohmic_model="no_loss", + ) + assert model.config is not None + + ac_power_and_pv_capacity_global = model.estimate( + years=years, months=months, xs=xs, ys=ys + ) + assert ac_power_and_pv_capacity_global is not None + assert isinstance( + ac_power_and_pv_capacity_global, (xr.DataArray, xr.Dataset) + ) + assert list(ac_power_and_pv_capacity_global.dims) == ["time", "x", "y"] + + wind_ds_cls = load_dataset("wind_3d_hourly_test") + wind_ds = wind_ds_cls(years=years, months=months) + assert wind_ds.downloaded, "Wind fixture NetCDF should be present" + + wxs, wys = _fixture_xy_slices(wind_ds) + + wind_model = WindInterpolationModel(wind_ds) + wind_model.prepare() + + wind_speed = wind_model.estimate( + years=years, months=months, xs=wxs, ys=wys, height=12 + ) + assert list(wind_speed.dims) == ["time", "x", "y"] + assert ( + "valid_time" not in wind_speed.dims + and "valid_time" not in wind_speed.coords + ) diff --git a/tests/pr/test_merra2.py b/tests/pr/test_merra2.py index ffff5169..2563f13d 100644 --- a/tests/pr/test_merra2.py +++ b/tests/pr/test_merra2.py @@ -13,50 +13,50 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import logging +# import logging -from geodata.datasets import DatasetType, load_dataset +# from geodata.datasets import DatasetType, load_dataset -logging.basicConfig(level=logging.INFO) +# logging.basicConfig(level=logging.INFO) -def get_data_configs() -> list[str]: - return [ - "surface_flux_monthly", - "slv_radiation_monthly", - "surface_flux_hourly", - "slv_radiation_hourly", - ] +# def get_data_configs() -> list[str]: +# return [ +# "surface_flux_monthly", +# "slv_radiation_monthly", +# "surface_flux_hourly", +# "slv_radiation_hourly", +# ] -def get_bounds() -> list[list[int]]: - return [[30, -10, 60, 10]] +# def get_bounds() -> list[list[int]]: +# return [[30, -10, 60, 10]] -def get_years() -> list[slice]: - return [slice(2005, 2005)] +# def get_years() -> list[slice]: +# return [slice(2005, 2005)] -def get_months() -> list[slice]: - return [slice(1, 1)] +# def get_months() -> list[slice]: +# return [slice(1, 1)] -def get_merra2(data_config: str, bound: list[int], year: slice, month: slice): - dataset_cls = load_dataset(data_config) - dataset: DatasetType = dataset_cls( - years=year, months=month, bounds=bound, testing=True - ) - if not dataset.downloaded: - dataset.download() - return dataset +# def get_merra2(data_config: str, bound: list[int], year: slice, month: slice): +# dataset_cls = load_dataset(data_config) +# dataset: DatasetType = dataset_cls( +# years=year, months=month, bounds=bound, testing=True +# ) +# if not dataset.downloaded: +# dataset.download() +# return dataset -def test_download(): - configs = get_data_configs() - years = get_years() - months = get_months() - bounds = get_bounds() +# def test_download(): +# configs = get_data_configs() +# years = get_years() +# months = get_months() +# bounds = get_bounds() - for config, year, month, bound in zip(configs, years, months, bounds): - dataset = get_merra2(config, bound, year, month) - assert dataset.downloaded +# for config, year, month, bound in zip(configs, years, months, bounds): +# dataset = get_merra2(config, bound, year, month) +# assert dataset.downloaded diff --git a/tests/pr/test_wind_xarraymask_integration.py b/tests/pr/test_wind_xarraymask_integration.py new file mode 100644 index 00000000..d509e369 --- /dev/null +++ b/tests/pr/test_wind_xarraymask_integration.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import numpy as np +import rasterio as ras +import xarray as xr +from dask.distributed import Client +from rasterio.transform import from_bounds + +from geodata import XarrayMask +from geodata.datasets import load_dataset +from geodata.model.wind import WindInterpolationModel + + +def _create_saved_mask_from_output_grid( + output: xr.DataArray, + mask_dir: Path, + name: str = "wind_xmask", +) -> None: + x = output["x"].values + y = output["y"].values + + lon = np.sort(np.asarray(x, dtype=float)) + lat = np.sort(np.asarray(y, dtype=float)) + lon_step = float(np.abs(lon[1] - lon[0])) + lat_step = float(np.abs(lat[1] - lat[0])) + west = float(lon.min() - lon_step / 2) + east = float(lon.max() + lon_step / 2) + south = float(lat.min() - lat_step / 2) + north = float(lat.max() + lat_step / 2) + + nlon_hi = len(lon) * 2 + nlat_hi = len(lat) * 2 + transform = from_bounds(west, south, east, north, nlon_hi, nlat_hi) + + arr = np.zeros((nlat_hi, nlon_hi), dtype=np.uint8) + arr[nlat_hi // 4 : 3 * nlat_hi // 4, nlon_hi // 4 : 3 * nlon_hi // 4] = 1 + + source_tif = mask_dir / "source.tif" + with ras.open( + str(source_tif), + "w", + driver="GTiff", + height=arr.shape[0], + width=arr.shape[1], + count=1, + dtype=arr.dtype, + compress="lzw", + crs="+proj=latlong", + transform=transform, + ) as dst: + dst.write(arr, 1) + + from geodata import Mask + + mask = Mask(name=name, mask_dir=str(mask_dir)) + mask.add_layer(str(source_tif), layer_name="source") + mask.merge_layer(show_raster=False) + mask.save_mask() + + +def test_wind_estimate_with_xarray_mask_offline(tmp_path): + years = slice(2016, 2016) + months = slice(1, 1) + + with Client(processes=True, threads_per_worker=1): + ds_cls = load_dataset("wind_3d_hourly_test") + ds = ds_cls(years=years, months=months) + assert ds.downloaded, "Wind fixture NetCDF should be present" + + model = WindInterpolationModel(ds) + model.prepare(force=True) + + base = model.estimate(years=years, months=months, height=12) + assert isinstance(base, xr.DataArray) + + mask_dir = tmp_path / "masks" + mask_dir.mkdir(parents=True, exist_ok=True) + mask_name = "wind_xmask" + _create_saved_mask_from_output_grid(base, mask_dir, name=mask_name) + + base_ds = base.to_dataset(name=base.name or "value") + xmask = XarrayMask.from_name(mask_name, grid=base_ds, mask_dir=str(mask_dir)) + masked = xmask.apply( + base_ds, + mode="where", + include_area=True, + ) + + assert isinstance(masked, dict) + assert set(masked.keys()) == {"merged_mask"} + + merged = masked["merged_mask"] + assert "area" in merged + value_vars = [v for v in merged.data_vars if v not in {"area"}] + assert len(value_vars) == 1 + var = value_vars[0] + + attached = xmask.attach(base, include_area=False)["merged_mask"] + valid = attached["mask"] > 0 + expected = attached[var].where(valid) + xr.testing.assert_allclose(merged[var], expected)