Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ jobs:
env:
HYDROWEB_API_KEY: ${{ secrets.HYDROWEB_API_KEY }}
EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY: ${{ secrets.EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY }}
EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }}
run: |
if [ "${{ inputs.download-test }}" = "true" ]; then
pytest pixcdust/tests --dl --realdata -v -rs --cov=pixcdust --cov-report=term-missing --cov-report=html --cov-report=markdown:coverage.md
Expand All @@ -56,3 +57,19 @@ jobs:
run: |
echo "## Code coverage" >> "$GITHUB_STEP_SUMMARY"
cat coverage.md >> "$GITHUB_STEP_SUMMARY"

- name: Test notebooks
env:
HYDROWEB_API_KEY: ${{ secrets.HYDROWEB_API_KEY }}
EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY: ${{ secrets.EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY }}
EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }}
run: |
mkdir -p /tmp/notebooks

papermill pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb /tmp/notebooks/download_pixc_to_gpkg
papermill pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb /tmp/notebooks/download_pixc_to_zarr
papermill pixcdust/notebooks/convert/convert_to_zcollection.ipynb /tmp/notebooks/convert_to_zcollection
papermill pixcdust/notebooks/convert/dggs_tuto.ipynb /tmp/notebooks/dggs_tuto
papermill pixcdust/notebooks/read_and_use/read_netcdf.ipynb /tmp/notebooks/read_netcdf


104 changes: 104 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
name: Publish to PyPI

on:
push:
tags:
- "v*"
workflow_dispatch:


permissions:
contents: read

jobs:
build:
name: Test and build distributions
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"

# This step check cohenrece between tags version and pyproject.toml version
- name: Verify version
if: startsWith(github.ref, 'refs/tags/')
env:
RELEASE_TAG: ${{ github.ref_name }}
run: |
python - <<'PY'
import os
import tomllib
from pathlib import Path

project = tomllib.loads(
Path("pyproject.toml").read_text(encoding="utf-8")
)["project"]

package_version = project["version"]
tag_version = os.environ["RELEASE_TAG"].removeprefix("v")

if package_version != tag_version:
raise SystemExit(
f"Tag v{tag_version} != pyproject.toml"
f"version {package_version}"
)

print(f"Publishing version {package_version}")
PY

# Install dependencies and launch linter, test, build and check distribution
- name: Install dependencies
run: |
pip install --upgrade pip
pip install --upgrade build
pip install ".[dev]"

- name: Run Ruff linter
run: ruff check .

- name: Run Ruff formatter check
run: ruff format --check .

- name: Test
run: pytest pixcdust/tests --realdata -v -rs

- name: Build distributions
run: python -m build

- name: Check distributions
run: python -m twine check dist/*

- name: Upload distributions
uses: actions/upload-artifact@v4
with:
name: python-distributions
path: dist/
if-no-files-found: error

publish:
name: Publish to PyPI
needs: build
runs-on: ubuntu-latest

environment:
name: pypi
url: https://pypi.org/project/pixcdusttest/

permissions:
id-token: write

steps:
- name: Download distributions
uses: actions/download-artifact@v4
with:
name: python-distributions
path: dist/

- name: Publish distributions
uses: pypa/gh-action-pypi-publish@release/v1

168 changes: 168 additions & 0 deletions pixcdust/downloaders/podaac.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#
# Copyright (C) 2024 Centre National d'Etudes Spatiales (CNES)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Downloaders for podacc. Require an API-Key see HELP_MESSAGE."""

import os
from datetime import date, datetime, time
from pathlib import Path

import earthaccess
import shapely
from shapely.geometry import shape

from pixcdust.downloaders.hydroweb_next import Downloader

HELP_MESSAGE = """
Download products from podaac
"""


class PodaacDownloader(Downloader):
"""Downloader for SWOT PIXC products from NASA PO.DAAC."""

PROVIDER = "podaac"

def __init__(self, *args, **kwargs):
"""Downloader for SWOT Pixel Cloud files from podaac.

Keyword Args:
geometry: A geometry used as search criteria. Defaults to None.
dates: Minimum and maximum dates to be used as search criteria.
Defaults to None.
path_download:
download path. Defaults to "/tmp/hydroweb_next".
verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG).
Defaults to 0.

Raises:
AttributeError: if the geometry is not one
of (str, tuple, list, geopandas.GeoDataFrame)
"""
super().__init__(*args, **kwargs)

def setup(self) -> None:
"""Initialize the Earthdata session."""
# Test if token is ok
token = os.environ.get("EARTHDATA_TOKEN")

if not token:
raise RuntimeError(
"Earthdata token required. "
"Please set the EARTHDATA_TOKEN environment variable."
)

self.auth = earthaccess.login(strategy="environment")
self.downloaded_paths: list[str] = []

def define_query(self) -> dict:
"""Build temporal search parameters."""
query = {}

if self.dates is not None:
start_date, end_date = self.dates
query["temporal"] = (
self._format_date(start_date, end=False),
self._format_date(end_date, end=True),
)

return query

@staticmethod
def _format_date(value: date | datetime, *, end: bool) -> str:
"""Format a date for Earthaccess temporal searches."""
if isinstance(value, datetime):
return value.isoformat(timespec="seconds")

boundary = time.max if end else time.min
return datetime.combine(value, boundary).isoformat(timespec="seconds")

def _search(self, geom=None) -> None:
"""Search granules intersecting the requested geometry."""

kwargs = {
"short_name": self.collection_name,
**self.query_args,
}

if geom is not None:
geometry = shape(geom) if isinstance(geom, dict) else shapely.from_wkt(geom)

# earthaccess expects coordinates in longitude/latitude.
if not geometry.is_empty:
if geometry.geom_type == "Polygon":
kwargs["polygon"] = list(geometry.exterior.coords)
elif geometry.geom_type == "MultiPolygon":
# Search each polygon separately; search_download()
# already handles GeoDataFrames one geometry at a time.
kwargs["polygon"] = list(
max(
geometry.geoms, key=lambda polygon: polygon.area
).exterior.coords
)
else:
raise ValueError(f"Unsupported geometry type: {geometry.geom_type}")

results = earthaccess.search_data(**kwargs)

# A GeoDataFrame may contain multiple polygons. Avoid duplicates.
existing_ids = {
result["meta"]["concept-id"]
for result in self.search_results
if isinstance(result, dict) and "meta" in result
}

for result in results:
metadata = result.get("meta", {}) if isinstance(result, dict) else {}
concept_id = metadata.get("concept-id")

if concept_id is None or concept_id not in existing_ids:
self.search_results.append(result)
if concept_id is not None:
existing_ids.add(concept_id)

def _download(self) -> list[str]:
"""Download the matching granules and return local NetCDF paths."""
if not self.search_results:
return []

paths = earthaccess.download(
self.search_results,
local_path=self.path_download,
)

# Filter on .nc file
self.downloaded_paths = [
str(Path(path)) for path in paths if Path(path).suffix.lower() == ".nc"
]


def PodaacPixCDownloader(*args, **kwargs):
"""Downloader for SWOT Pixel Cloud files from PO.DAAC

Keyword Args:
geometry: A geometry used as search criteria. Defaults to None.
dates: Minimum and maximum dates to be used as search criteria.
Defaults to None.
path_download:
download path. Defaults to "/tmp/hydroweb_next".
verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG).
Defaults to 0.

Raises:
AttributeError: if the geometry is not one
of (str, tuple, list, geopandas.GeoDataFrame)
"""
return PodaacDownloader("SWOT_L2_HR_PIXC_D", *args, **kwargs)
Loading
Loading