Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ jobs:
env:
HYDROWEB_API_KEY: ${{ secrets.HYDROWEB_API_KEY }}
EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY: ${{ secrets.EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY }}
EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }}
run: |
if [ "${{ inputs.download-test }}" = "true" ]; then
pytest pixcdust/tests --dl --realdata -v -rs --cov=pixcdust --cov-report=term-missing --cov-report=html --cov-report=markdown:coverage.md
Expand All @@ -56,3 +57,5 @@ jobs:
run: |
echo "## Code coverage" >> "$GITHUB_STEP_SUMMARY"
cat coverage.md >> "$GITHUB_STEP_SUMMARY"


44 changes: 44 additions & 0 deletions .github/workflows/ci_notebook.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: NotebookTest

on:
push:
branches: ["**"]
workflow_dispatch:


jobs:
notebook-test:
name: Lint & Tests check
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v5

- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
cache: "pip"

- name: Install dependencies
run: |
pip install --upgrade pip
pip install --upgrade build
pip install ".[dev]"

- name: Test notebooks
env:
HYDROWEB_API_KEY: ${{ secrets.HYDROWEB_API_KEY }}
EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY: ${{ secrets.EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY }}
EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }}
run: |
mkdir -p /tmp/notebooks

papermill pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb /tmp/notebooks/download_pixc_to_gpkg
papermill pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb /tmp/notebooks/download_pixc_to_zarr
papermill pixcdust/notebooks/convert/convert_to_zcollection.ipynb /tmp/notebooks/convert_to_zcollection
papermill pixcdust/notebooks/convert/dggs_tuto.ipynb /tmp/notebooks/dggs_tuto
papermill pixcdust/notebooks/read_and_use/read_netcdf.ipynb /tmp/notebooks/read_netcdf


104 changes: 104 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
name: Publish to PyPI

on:
push:
tags:
- "v*"
workflow_dispatch:


permissions:
contents: read

jobs:
build:
name: Test and build distributions
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"

# This step check cohenrece between tags version and pyproject.toml version
- name: Verify version
if: startsWith(github.ref, 'refs/tags/')
env:
RELEASE_TAG: ${{ github.ref_name }}
run: |
python - <<'PY'
import os
import tomllib
from pathlib import Path

project = tomllib.loads(
Path("pyproject.toml").read_text(encoding="utf-8")
)["project"]

package_version = project["version"]
tag_version = os.environ["RELEASE_TAG"].removeprefix("v")

if package_version != tag_version:
raise SystemExit(
f"Tag v{tag_version} != pyproject.toml"
f"version {package_version}"
)

print(f"Publishing version {package_version}")
PY

# Install dependencies and launch linter, test, build and check distribution
- name: Install dependencies
run: |
pip install --upgrade pip
pip install --upgrade build
pip install ".[dev]"

- name: Run Ruff linter
run: ruff check .

- name: Run Ruff formatter check
run: ruff format --check .

- name: Test
run: pytest pixcdust/tests --realdata -v -rs

- name: Build distributions
run: python -m build

- name: Check distributions
run: python -m twine check dist/*

- name: Upload distributions
uses: actions/upload-artifact@v4
with:
name: python-distributions
path: dist/
if-no-files-found: error

publish:
name: Publish to PyPI
needs: build
runs-on: ubuntu-latest

environment:
name: pypi
url: https://pypi.org/project/pixcdusttest/

permissions:
id-token: write

steps:
- name: Download distributions
uses: actions/download-artifact@v4
with:
name: python-distributions
path: dist/

- name: Publish distributions
uses: pypa/gh-action-pypi-publish@release/v1

168 changes: 168 additions & 0 deletions pixcdust/downloaders/podaac.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#
# Copyright (C) 2024 Centre National d'Etudes Spatiales (CNES)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Downloaders for podacc. Require an API-Key see HELP_MESSAGE."""

import os
from datetime import date, datetime, time
from pathlib import Path

import earthaccess
import shapely
from shapely.geometry import shape

from pixcdust.downloaders.hydroweb_next import Downloader

HELP_MESSAGE = """
Download products from podaac
"""


class PodaacDownloader(Downloader):
"""Downloader for SWOT PIXC products from NASA PO.DAAC."""

PROVIDER = "podaac"

def __init__(self, *args, **kwargs):
"""Downloader for SWOT Pixel Cloud files from podaac.

Keyword Args:
geometry: A geometry used as search criteria. Defaults to None.
dates: Minimum and maximum dates to be used as search criteria.
Defaults to None.
path_download:
download path. Defaults to "/tmp/hydroweb_next".
verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG).
Defaults to 0.

Raises:
AttributeError: if the geometry is not one
of (str, tuple, list, geopandas.GeoDataFrame)
"""
super().__init__(*args, **kwargs)

def setup(self) -> None:
"""Initialize the Earthdata session."""
# Test if token is ok
token = os.environ.get("EARTHDATA_TOKEN")

if not token:
raise RuntimeError(
"Earthdata token required. "
"Please set the EARTHDATA_TOKEN environment variable."
)

self.auth = earthaccess.login(strategy="environment")
self.downloaded_paths: list[str] = []

def define_query(self) -> dict:
"""Build temporal search parameters."""
query = {}

if self.dates is not None:
start_date, end_date = self.dates
query["temporal"] = (
self._format_date(start_date, end=False),
self._format_date(end_date, end=True),
)

return query

@staticmethod
def _format_date(value: date | datetime, *, end: bool) -> str:
"""Format a date for Earthaccess temporal searches."""
if isinstance(value, datetime):
return value.isoformat(timespec="seconds")

boundary = time.max if end else time.min
return datetime.combine(value, boundary).isoformat(timespec="seconds")

def _search(self, geom=None) -> None:
"""Search granules intersecting the requested geometry."""

kwargs = {
"short_name": self.collection_name,
**self.query_args,
}

if geom is not None:
geometry = shape(geom) if isinstance(geom, dict) else shapely.from_wkt(geom)

# earthaccess expects coordinates in longitude/latitude.
if not geometry.is_empty:
if geometry.geom_type == "Polygon":
kwargs["polygon"] = list(geometry.exterior.coords)
elif geometry.geom_type == "MultiPolygon":
# Search each polygon separately; search_download()
# already handles GeoDataFrames one geometry at a time.
kwargs["polygon"] = list(
max(
geometry.geoms, key=lambda polygon: polygon.area
).exterior.coords
)
else:
raise ValueError(f"Unsupported geometry type: {geometry.geom_type}")

results = earthaccess.search_data(**kwargs)

# A GeoDataFrame may contain multiple polygons. Avoid duplicates.
existing_ids = {
result["meta"]["concept-id"]
for result in self.search_results
if isinstance(result, dict) and "meta" in result
}

for result in results:
metadata = result.get("meta", {}) if isinstance(result, dict) else {}
concept_id = metadata.get("concept-id")

if concept_id is None or concept_id not in existing_ids:
self.search_results.append(result)
if concept_id is not None:
existing_ids.add(concept_id)

def _download(self) -> list[str]:
"""Download the matching granules and return local NetCDF paths."""
if not self.search_results:
return []

paths = earthaccess.download(
self.search_results,
local_path=self.path_download,
)

# Filter on .nc file
self.downloaded_paths = [
str(Path(path)) for path in paths if Path(path).suffix.lower() == ".nc"
]


def PodaacPixCDownloader(*args, **kwargs):
"""Downloader for SWOT Pixel Cloud files from PO.DAAC

Keyword Args:
geometry: A geometry used as search criteria. Defaults to None.
dates: Minimum and maximum dates to be used as search criteria.
Defaults to None.
path_download:
download path. Defaults to "/tmp/hydroweb_next".
verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG).
Defaults to 0.

Raises:
AttributeError: if the geometry is not one
of (str, tuple, list, geopandas.GeoDataFrame)
"""
return PodaacDownloader("SWOT_L2_HR_PIXC_D", *args, **kwargs)
Loading
Loading