From d1afdac393c3212e1d24f3186166150d0cdc4d1a Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Mon, 26 Oct 2026 14:06:34 +0100 Subject: [PATCH 01/13] [DOC] maj doc --- .github/workflows/ci_notebook.yml | 2 +- README.md | 4 ++-- doc/index.rst | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_notebook.yml b/.github/workflows/ci_notebook.yml index ad86626..052767b 100644 --- a/.github/workflows/ci_notebook.yml +++ b/.github/workflows/ci_notebook.yml @@ -8,7 +8,7 @@ on: jobs: notebook-test: - name: Lint & Tests check + name: Notebook check runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 1410027..ba125f1 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,9 @@ python tests/init_tests.py -I INPUT_FOLDER -H HYDROWEB_AUTH You then can run the converters tests: ``` -pytest +pytest pixcdust/tests --realdata ``` You can also run all the tests including the downloaders tests with: ``` -pytest --dl +pytest pixcdust/tests --realdata --dl ``` diff --git a/doc/index.rst b/doc/index.rst index 225f2e2..420eeaf 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -7,7 +7,7 @@ pixcdust documentation ====================== 🎩 🪄 A Pinch of PixC Dust 🐇 ----------------------------- +----------------------------- This python project centralizes librairies to facilitate local studies based on SWOT-HR Level-2 Pixel Cloud products. @@ -34,9 +34,9 @@ Start here to understand what you can do: :doc:`tutorials`. ---------------- The downloader classes allow you to directly download SWOT Pixel Cloud files from hydroweb.next (or other sources such as PO.DAAC to be implemented). For hydroweb.next, it requires you to create an account and an API Key (token) from the platform: https://hydroweb.next.theia-land.fr. -Then, carefully store your API-Key either : - * in your eodag configuration file (usually `~/.config/eodag/eodag.yml`, automatically generated the first time you use eodag) in ``auth/credentials/apikey="PLEASE_CHANGE_ME"`` - * in an environment variable ``export EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY="PLEASE_CHANGE_ME"`` +Then, carefully store your API-Key either + * in your eodag configuration file: (usually `~/.config/eodag/eodag.yml`, automatically generated the first time you use eodag) in ``auth/credentials/apikey="PLEASE_CHANGE_ME"`` + * in an environment variable: ``export EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY="PLEASE_CHANGE_ME"`` 🪄 *Converters* --------------- @@ -56,11 +56,11 @@ Here are some python script implementing the classes. 🔶 *Discrete Global Grid System (experimental)* ----------------------------------------------- -I enjoy DDGS a lot. It is pretty great if you want to perform on-the-fly "rasterization", partitionning, comparing pixels over time or space, etc. +I enjoy DGGS a lot. It is pretty great if you want to perform on-the-fly "rasterization", partitionning, comparing pixels over time or space, etc. Currently H3 and HEALPix are implemented. ✅ *Tests* ---------- +---------- You first need to configure the tests and download the tests data with init_tests.py. We recommend setting the following options: * INPUT_FOLDER is where the tests data will be downloaded (or are already available). * HYDROWEB_AUTH is your hydroweb.next API key. It is required to automatically download the tests data or run the downloaders tests. @@ -68,11 +68,11 @@ You first need to configure the tests and download the tests data with init_test ``python tests/init_tests.py -I INPUT_FOLDER -H HYDROWEB_AUTH`` You then can run the converters tests: -``pytest`` +``pytest pixcdust/tests --realdata`` You can also run all the tests including the downloaders tests with: -``pytest --dl`` +``pytest pixcdust/tests --realdata --dl`` .. toctree:: From 0ba44f03a251c3f257235b6930dcbc56ff2075b2 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 12:13:16 +0100 Subject: [PATCH 02/13] [ENH] Remove zcollection replaced by zarr --- pixcdust/converters/zarr.py | 66 +++++++++++++------------- pixcdust/readers/zarr.py | 59 +++++++++++++---------- pixcdust/tests/test_converters_mock.py | 2 +- 3 files changed, 68 insertions(+), 59 deletions(-) diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index 3a7b68f..eff96c9 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -21,17 +21,19 @@ from pathlib import Path import dask +import dask.distributed import dask.utils import fsspec import geopandas as gpd -import zcollection -import zcollection.indexing - +import pandas +import xarray + from pixcdust.converters.core import Converter from pixcdust.readers.netcdf import NcSimpleConstants, NcSimpleReader TIME_VARNAME = "time" - +# TODO is there a better solution than fix the chunk size here ? +CHUNK_SIZE = dask.utils.parse_bytes("2MiB") class Nc2ZarrConverter(Converter): """Converter from official SWOT Pixel Cloud Netcdf to Shapefile database @@ -74,7 +76,6 @@ def __init__( area_of_interest=area_of_interest, conditions=conditions, ) - self.collection: zcollection.collection.Collection = None self.__time_varname: str = TIME_VARNAME self.__fs = fsspec.filesystem("file") self.__chunk_size = dask.utils.parse_bytes("2MiB") @@ -85,6 +86,9 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: if mode in ["o", "overwrite"] and os.path.exists(path_out): shutil.rmtree(path_out) + path_out = Path(path_out) + append = path_out.exists() + with ( dask.distributed.LocalCluster(processes=True) as cluster, dask.distributed.Client(cluster) as _, @@ -106,41 +110,35 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: ) time = xr_ds_xarray["time"].values - import pandas - import xarray + time2 = pandas.DatetimeIndex(time).tz_localize(None).to_numpy() - xr_ds_xarray["time"] = xarray.DataArray(time2, dims=("points",)) - zc_ds = zcollection.Dataset.from_xarray(xr_ds_xarray) - - # zc_ds = zcollection.Dataset.from_xarray( - # xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), - # ) - zc_ds.block_size_limit = self.__chunk_size - zc_ds.chunks = {next(iter(zc_ds.dimensions.keys())): self.__chunk_size} - - init = True - if not os.path.exists(path_out) and init: - partition_handler = zcollection.partitioning.Date( - (xr_ds.cst.default_added_time_name,), - "s", + xr_ds_xarray = xr_ds_xarray.assign_coords(time=("points", time2)) + + # Approximate the original 2 MiB chunk-size target. + npoints = xr_ds_xarray.sizes.get("points", 0) + if npoints: + bytes_per_point = max(1, xr_ds_xarray.nbytes // npoints) + chunk_points = max(1, CHUNK_SIZE // bytes_per_point) + xr_ds_xarray = xr_ds_xarray.chunk( + {"points": int(chunk_points)} ) - self.collection = zcollection.create_collection( - axis=self.__time_varname, - ds=zc_ds, - partition_handler=partition_handler, - partition_base_dir=path_out, - filesystem=self.__fs, + if append: + xr_ds_xarray.to_zarr( + path_out, + mode="a", + append_dim="points", + consolidated=True, + zarr_version=2, ) - init = False - else: - self.collection = zcollection.open_collection( + xr_ds_xarray.to_zarr( path_out, - filesystem=self.__fs, mode="w", + consolidated=True, + zarr_version=2, ) - self.collection.insert( - zc_ds, merge_callable=zcollection.collection.merging.merge_time_series - ) + + #xr_ds.close() + diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index ee8fd66..6c6f9e6 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -18,7 +18,7 @@ import datetime import xarray as xr -import zcollection +import numpy as np from pixcdust.readers.base_reader import BaseReader @@ -50,33 +50,44 @@ def read( Only load data dated within the interval. """ - collection = zcollection.open_collection( + self.data = xr.open_zarr( self.path, - mode="r", ) - if date_interval: + if date_interval is not None: date_min = date_interval[0] date_max = date_interval[1] - # LOCALE timezone - data_z = collection.load( - filters=lambda keys: ( - date_min - <= datetime.datetime( - keys["year"], - keys["month"], - keys["day"], - keys["hour"], - keys["minute"], - keys["second"], - tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo, - ) - <= date_max + + + if "time" not in self.data: + self.data.close() + self.data = xr.Dataset() + raise ValueError( + "Cannot filter the Zarr dataset by date: " + "no 'time' coordinate is available." ) + + # Convert timezone-aware bounds to local naive timestamps, + # matching the local-time interpretation of the old reader. + if date_min.tzinfo is not None: + date_min = date_min.astimezone().replace(tzinfo=None) + + if date_max.tzinfo is not None: + date_max = date_max.astimezone().replace(tzinfo=None) + + date_min = np.datetime64( + date_min.replace(tzinfo=None), "ns" + ) + date_max = np.datetime64( + date_max.replace(tzinfo=None), "ns" ) - else: - data_z = collection.load() - if data_z is None: - self.data = xr.Dataset() - else: - self.data = data_z.to_xarray() + + mask = ( + (self.data["time"] >= date_min) + & (self.data["time"] <= date_max) + ) + mask = mask.compute() + + self.data = self.data.isel(points=mask) + + diff --git a/pixcdust/tests/test_converters_mock.py b/pixcdust/tests/test_converters_mock.py index 2c54310..53b8f67 100644 --- a/pixcdust/tests/test_converters_mock.py +++ b/pixcdust/tests/test_converters_mock.py @@ -27,7 +27,7 @@ def test_convert_ds_to_gpkg(self): def test_convert_ds_to_zarr(self): """function for testing the conversion from - netcdf to zarr with zcollection + netcdf to zarr """ pixc = Nc2ZarrConverter( From 4ba7c18d69fcb0aa4f920dc9a20cebc0fce1f744 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 12:13:35 +0100 Subject: [PATCH 03/13] [DOC] Remove zcollection replaced by zarr --- README.md | 2 +- doc/index.rst | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba125f1..0ff1ffc 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Then, carefully store your API-Key : ## 🪄 *Converters* The converter classes allow you to create more easy-to-use databases than the original netcdf4 format. The various databases are designed for **local** studies, not for huge country-scale databases (though it should work, they will not be efficient). -Zarr (with zcollection), geopackage and shapefile are currently supported. +Zarr, geopackage and shapefile are currently supported. The converters allow you to limit the databases to areas of interest (provided by polygons) and variables of interest (limitated to the pixel_cloud group mono-dimensional variables). Users are encouraged to limit the number of variables to what is useful, especially for geopackage format, but also for the planet ;) diff --git a/doc/index.rst b/doc/index.rst index 420eeaf..4bd6012 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -41,7 +41,7 @@ Then, carefully store your API-Key either 🪄 *Converters* --------------- The converter classes allow you to create more easy-to-use databases than the original netcdf4 format. The various databases are designed for **local** studies, not for huge country-scale databases (though it should work, they will not be efficient). -Zarr (with zcollection), geopackage and shapefile are currently supported. +Zarr, geopackage and shapefile are currently supported. The converters allow you to limit the databases to areas of interest (provided by polygons) and variables of interest (limitated to the pixel_cloud group mono-dimensional variables). Users are encouraged to limit the number of variables to what is useful, especially for geopackage format, but also for the planet ;) @@ -68,10 +68,12 @@ You first need to configure the tests and download the tests data with init_test ``python tests/init_tests.py -I INPUT_FOLDER -H HYDROWEB_AUTH`` You then can run the converters tests: + ``pytest pixcdust/tests --realdata`` You can also run all the tests including the downloaders tests with: + ``pytest pixcdust/tests --realdata --dl`` From e08a7e7e32197b83c0d8da9efed2b32417d0a234 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 12:19:29 +0100 Subject: [PATCH 04/13] [ENH] Remove zcollection replaced by zarr --- pixcdust/converters/zarr.py | 1 - pixcdust/readers/zarr.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index eff96c9..485faff 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -26,7 +26,6 @@ import fsspec import geopandas as gpd import pandas -import xarray from pixcdust.converters.core import Converter from pixcdust.readers.netcdf import NcSimpleConstants, NcSimpleReader diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index 6c6f9e6..e04402b 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -17,8 +17,8 @@ import datetime -import xarray as xr import numpy as np +import xarray as xr from pixcdust.readers.base_reader import BaseReader From be7a9e36a55ccd4f4ec4fe844731eecbdd734a48 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 13:35:59 +0100 Subject: [PATCH 05/13] [ENH] Remove zcollection replaced by zarr --- pixcdust/converters/zarr.py | 13 ++++--------- pixcdust/readers/zarr.py | 20 +++++--------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index 485faff..d0037b1 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -26,7 +26,7 @@ import fsspec import geopandas as gpd import pandas - + from pixcdust.converters.core import Converter from pixcdust.readers.netcdf import NcSimpleConstants, NcSimpleReader @@ -34,6 +34,7 @@ # TODO is there a better solution than fix the chunk size here ? CHUNK_SIZE = dask.utils.parse_bytes("2MiB") + class Nc2ZarrConverter(Converter): """Converter from official SWOT Pixel Cloud Netcdf to Shapefile database @@ -87,7 +88,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: path_out = Path(path_out) append = path_out.exists() - + with ( dask.distributed.LocalCluster(processes=True) as cluster, dask.distributed.Client(cluster) as _, @@ -110,7 +111,6 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: time = xr_ds_xarray["time"].values - time2 = pandas.DatetimeIndex(time).tz_localize(None).to_numpy() xr_ds_xarray = xr_ds_xarray.assign_coords(time=("points", time2)) @@ -119,9 +119,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: if npoints: bytes_per_point = max(1, xr_ds_xarray.nbytes // npoints) chunk_points = max(1, CHUNK_SIZE // bytes_per_point) - xr_ds_xarray = xr_ds_xarray.chunk( - {"points": int(chunk_points)} - ) + xr_ds_xarray = xr_ds_xarray.chunk({"points": int(chunk_points)}) if append: xr_ds_xarray.to_zarr( @@ -138,6 +136,3 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: consolidated=True, zarr_version=2, ) - - #xr_ds.close() - diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index e04402b..88b4a88 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -57,8 +57,7 @@ def read( if date_interval is not None: date_min = date_interval[0] date_max = date_interval[1] - - + if "time" not in self.data: self.data.close() self.data = xr.Dataset() @@ -75,19 +74,10 @@ def read( if date_max.tzinfo is not None: date_max = date_max.astimezone().replace(tzinfo=None) - date_min = np.datetime64( - date_min.replace(tzinfo=None), "ns" - ) - date_max = np.datetime64( - date_max.replace(tzinfo=None), "ns" - ) - - mask = ( - (self.data["time"] >= date_min) - & (self.data["time"] <= date_max) - ) + date_min = np.datetime64(date_min.replace(tzinfo=None), "ns") + date_max = np.datetime64(date_max.replace(tzinfo=None), "ns") + + mask = (self.data["time"] >= date_min) & (self.data["time"] <= date_max) mask = mask.compute() self.data = self.data.isel(points=mask) - - From ee20dea594890c7f4d9d4449dc18b8b0d06b2db6 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 13:54:51 +0100 Subject: [PATCH 06/13] [ENH] Remove zcollection replaced by zarr --- pixcdust/readers/zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index 88b4a88..5f89465 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -73,7 +73,7 @@ def read( if date_max.tzinfo is not None: date_max = date_max.astimezone().replace(tzinfo=None) - + date_min = np.datetime64(date_min.replace(tzinfo=None), "ns") date_max = np.datetime64(date_max.replace(tzinfo=None), "ns") From 9ffe749233ca43b8143572919fb4fde22e3480b8 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 14:27:18 +0100 Subject: [PATCH 07/13] [ENH] Remove zcollection replaced by zarr --- .../convert/convert_to_zcollection.ipynb | 268 ++++++++---------- 1 file changed, 126 insertions(+), 142 deletions(-) diff --git a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb index 970be42..a13dc91 100644 --- a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb +++ b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb @@ -27,19 +27,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/cedric/projet/PixCDust/venv/lib/python3.12/site-packages/distributed/client.py:3429: UserWarning: Sending large graph of size 93.08 MiB.\n", - "This may cause some slowdown.\n", - "Consider loading the data with Dask directly\n", - " or using futures or delayed objects to embed the data into the graph without repetition.\n", - "See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.\n", - " warnings.warn(\n", - "/home/cedric/projet/PixCDust/venv/lib/python3.12/site-packages/distributed/client.py:3429: UserWarning: Sending large graph of size 93.08 MiB.\n", - "This may cause some slowdown.\n", - "Consider loading the data with Dask directly\n", - " or using futures or delayed objects to embed the data into the graph without repetition.\n", - "See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.\n", + "/home/cedric/projet/PixCDust/venv/lib/python3.12/site-packages/distributed/node.py:195: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 40669 instead\n", " warnings.warn(\n", - "/home/cedric/projet/PixCDust/venv/lib/python3.12/site-packages/distributed/client.py:3429: UserWarning: Sending large graph of size 93.08 MiB.\n", + "/home/cedric/projet/PixCDust/PixCDust/pixcdust/converters/zarr.py:133: FutureWarning: zarr_version is deprecated, use zarr_format\n", + " xr_ds_xarray.to_zarr(\n", + "/home/cedric/projet/PixCDust/venv/lib/python3.12/site-packages/distributed/client.py:3429: UserWarning: Sending large graph of size 163.11 MiB.\n", "This may cause some slowdown.\n", "Consider loading the data with Dask directly\n", " or using futures or delayed objects to embed the data into the graph without repetition.\n", @@ -82,22 +76,12 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2026-10-13 17:03:07,068 - distributed.scheduler - WARNING - Worker failed to heartbeat for 536s; removing: \n", - "2026-10-13 17:03:07,215 - distributed.scheduler - WARNING - Workers ['inproc://10.0.2.15/43568/4'] do not use a nanny and will be terminated without restarting them\n", - "2026-10-13 17:03:07,241 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'inproc://10.0.2.15/43568/4'.\n", - "2026-10-13 17:03:07,280 - distributed.worker - WARNING - Scheduler was unaware of this worker; shutting down.\n" - ] } ], "source": [ @@ -149,15 +133,15 @@ " \n", " \n", " \n", + " classification\n", + " cycle_number\n", + " height\n", " pass_number\n", - " longitude\n", - " tile_number\n", " sig0\n", - " time\n", - " height\n", - " cycle_number\n", + " tile_number\n", " latitude\n", - " classification\n", + " longitude\n", + " time\n", " \n", " \n", " points\n", @@ -175,63 +159,63 @@ " \n", " \n", " 0\n", - " 16\n", - " 0.744462\n", - " 78\n", - " -0.450141\n", - " 2023-04-06 07:46:18\n", - " 265.946594\n", - " 482\n", - " 43.768482\n", " 1.0\n", + " 482.0\n", + " 186.559509\n", + " 16.0\n", + " 0.117063\n", + " 78.0\n", + " 43.866646\n", + " 1.292987\n", + " 2023-04-06 07:46:18\n", " \n", " \n", " 1\n", - " 16\n", - " 0.744049\n", - " 78\n", - " -0.293464\n", - " 2023-04-06 07:46:18\n", - " 265.177887\n", - " 482\n", - " 43.768406\n", " 1.0\n", + " 482.0\n", + " 187.090530\n", + " 16.0\n", + " 0.326538\n", + " 78.0\n", + " 43.866718\n", + " 1.293394\n", + " 2023-04-06 07:46:18\n", " \n", " \n", " 2\n", - " 16\n", - " 0.743782\n", - " 78\n", - " 0.156394\n", - " 2023-04-06 07:46:18\n", - " 264.416779\n", - " 482\n", - " 43.768356\n", " 1.0\n", + " 482.0\n", + " 187.140320\n", + " 16.0\n", + " 0.435253\n", + " 78.0\n", + " 43.866749\n", + " 1.293555\n", + " 2023-04-06 07:46:18\n", " \n", " \n", " 3\n", - " 16\n", - " 0.742234\n", - " 78\n", - " 0.386425\n", - " 2023-04-06 07:46:18\n", - " 263.612488\n", - " 482\n", - " 43.768070\n", " 1.0\n", + " 482.0\n", + " 187.658203\n", + " 16.0\n", + " 0.094010\n", + " 78.0\n", + " 43.866795\n", + " 1.293808\n", + " 2023-04-06 07:46:18\n", " \n", " \n", " 4\n", - " 16\n", - " 0.741447\n", - " 78\n", - " 0.647449\n", - " 2023-04-06 07:46:18\n", - " 262.843506\n", - " 482\n", - " 43.767925\n", " 1.0\n", + " 482.0\n", + " 186.557190\n", + " 16.0\n", + " 0.435701\n", + " 78.0\n", + " 43.866779\n", + " 1.293738\n", + " 2023-04-06 07:46:18\n", " \n", " \n", " ...\n", @@ -246,100 +230,100 @@ " ...\n", " \n", " \n", - " 12200494\n", - " 16\n", - " 1.697695\n", - " 78\n", - " -0.893430\n", - " 2023-04-07 07:36:56\n", - " 294.755371\n", - " 483\n", - " 43.347736\n", + " 1639167\n", " 1.0\n", + " 483.0\n", + " 236.591522\n", + " 16.0\n", + " 0.899599\n", + " 78.0\n", + " 43.336502\n", + " 1.494740\n", + " 2023-04-07 07:36:56\n", " \n", " \n", - " 12200495\n", - " 16\n", - " 1.697800\n", - " 78\n", - " 0.203908\n", - " 2023-04-07 07:36:56\n", - " 294.738892\n", - " 483\n", - " 43.347752\n", + " 1639168\n", " 1.0\n", + " 483.0\n", + " 239.294220\n", + " 16.0\n", + " 0.458149\n", + " 78.0\n", + " 43.336620\n", + " 1.495401\n", + " 2023-04-07 07:36:56\n", " \n", " \n", - " 12200496\n", - " 16\n", - " 1.697667\n", - " 78\n", - " -0.141134\n", - " 2023-04-07 07:36:56\n", - " 293.052917\n", - " 483\n", - " 43.347729\n", + " 1639169\n", " 1.0\n", + " 483.0\n", + " 242.257080\n", + " 16.0\n", + " 0.405881\n", + " 78.0\n", + " 43.336750\n", + " 1.496110\n", + " 2023-04-07 07:36:56\n", " \n", " \n", - " 12200497\n", - " 16\n", - " 1.698264\n", - " 78\n", - " -0.110434\n", - " 2023-04-07 07:36:56\n", - " 296.487579\n", - " 483\n", - " 43.347836\n", + " 1639170\n", " 1.0\n", + " 483.0\n", + " 240.972733\n", + " 16.0\n", + " 0.338941\n", + " 78.0\n", + " 43.336731\n", + " 1.496008\n", + " 2023-04-07 07:36:56\n", " \n", " \n", - " 12200498\n", - " 16\n", - " 1.698193\n", - " 78\n", - " 0.922207\n", - " 2023-04-07 07:36:56\n", - " 295.234772\n", - " 483\n", - " 43.347824\n", + " 1639171\n", " 1.0\n", + " 483.0\n", + " 238.593582\n", + " 16.0\n", + " 0.676266\n", + " 78.0\n", + " 43.336674\n", + " 1.495697\n", + " 2023-04-07 07:36:56\n", " \n", " \n", "\n", - "

12200499 rows × 9 columns

\n", + "

1639172 rows × 9 columns

\n", "" ], "text/plain": [ - " pass_number longitude tile_number sig0 time \\\n", - "points \n", - "0 16 0.744462 78 -0.450141 2023-04-06 07:46:18 \n", - "1 16 0.744049 78 -0.293464 2023-04-06 07:46:18 \n", - "2 16 0.743782 78 0.156394 2023-04-06 07:46:18 \n", - "3 16 0.742234 78 0.386425 2023-04-06 07:46:18 \n", - "4 16 0.741447 78 0.647449 2023-04-06 07:46:18 \n", - "... ... ... ... ... ... \n", - "12200494 16 1.697695 78 -0.893430 2023-04-07 07:36:56 \n", - "12200495 16 1.697800 78 0.203908 2023-04-07 07:36:56 \n", - "12200496 16 1.697667 78 -0.141134 2023-04-07 07:36:56 \n", - "12200497 16 1.698264 78 -0.110434 2023-04-07 07:36:56 \n", - "12200498 16 1.698193 78 0.922207 2023-04-07 07:36:56 \n", + " classification cycle_number height pass_number sig0 \\\n", + "points \n", + "0 1.0 482.0 186.559509 16.0 0.117063 \n", + "1 1.0 482.0 187.090530 16.0 0.326538 \n", + "2 1.0 482.0 187.140320 16.0 0.435253 \n", + "3 1.0 482.0 187.658203 16.0 0.094010 \n", + "4 1.0 482.0 186.557190 16.0 0.435701 \n", + "... ... ... ... ... ... \n", + "1639167 1.0 483.0 236.591522 16.0 0.899599 \n", + "1639168 1.0 483.0 239.294220 16.0 0.458149 \n", + "1639169 1.0 483.0 242.257080 16.0 0.405881 \n", + "1639170 1.0 483.0 240.972733 16.0 0.338941 \n", + "1639171 1.0 483.0 238.593582 16.0 0.676266 \n", "\n", - " height cycle_number latitude classification \n", - "points \n", - "0 265.946594 482 43.768482 1.0 \n", - "1 265.177887 482 43.768406 1.0 \n", - "2 264.416779 482 43.768356 1.0 \n", - "3 263.612488 482 43.768070 1.0 \n", - "4 262.843506 482 43.767925 1.0 \n", - "... ... ... ... ... \n", - "12200494 294.755371 483 43.347736 1.0 \n", - "12200495 294.738892 483 43.347752 1.0 \n", - "12200496 293.052917 483 43.347729 1.0 \n", - "12200497 296.487579 483 43.347836 1.0 \n", - "12200498 295.234772 483 43.347824 1.0 \n", + " tile_number latitude longitude time \n", + "points \n", + "0 78.0 43.866646 1.292987 2023-04-06 07:46:18 \n", + "1 78.0 43.866718 1.293394 2023-04-06 07:46:18 \n", + "2 78.0 43.866749 1.293555 2023-04-06 07:46:18 \n", + "3 78.0 43.866795 1.293808 2023-04-06 07:46:18 \n", + "4 78.0 43.866779 1.293738 2023-04-06 07:46:18 \n", + "... ... ... ... ... \n", + "1639167 78.0 43.336502 1.494740 2023-04-07 07:36:56 \n", + "1639168 78.0 43.336620 1.495401 2023-04-07 07:36:56 \n", + "1639169 78.0 43.336750 1.496110 2023-04-07 07:36:56 \n", + "1639170 78.0 43.336731 1.496008 2023-04-07 07:36:56 \n", + "1639171 78.0 43.336674 1.495697 2023-04-07 07:36:56 \n", "\n", - "[12200499 rows x 9 columns]" + "[1639172 rows x 9 columns]" ] }, "execution_count": 5, From ecff2a91c5c279e2b5bb991ad2acef031b901e32 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 14:28:54 +0100 Subject: [PATCH 08/13] [TEST] Fix podaac test --- pixcdust/tests/test_downloaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixcdust/tests/test_downloaders.py b/pixcdust/tests/test_downloaders.py index 8dd2391..427e29f 100644 --- a/pixcdust/tests/test_downloaders.py +++ b/pixcdust/tests/test_downloaders.py @@ -54,7 +54,7 @@ def test_podaac_download(hydroweb_env, input_folder, tmp_folder): dl_dir = tmp_folder / "download_test" download_test_data_podaac(dl_dir) - dl_files = sorted(dl_dir.glob("**/*.nc")) + dl_files = sorted(dl_dir.glob("*.nc")) all_input_files = sorted(input_folder.glob("**/*.nc")) assert len(dl_files) == len(all_input_files) From d0cc8e5487033bb6b7a8594495963b1658ac3241 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 14:31:07 +0100 Subject: [PATCH 09/13] [ENH] dependencies remove zcollection python 3.13 compatibility --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb085dd..c4137cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,25 +14,26 @@ dependencies = [ "fiona>=1.9.5,<2.0.0", "pandas>=2.2.0,<3.0.0", "geopandas>=1.1.3,<2.0.0", - "h3>=3.7.6,<4.0.0", + "h3>=3.7.6,<4.3.0", "astropy>=8.0.0,<9.0.0", "astropy-healpix>=1.1.0,<2.0.0", "xarray>=2025.3.0,<2026.7.0", "shapely>=2.0.3,<3.0.0", "click>=8.1.7,<9.0.0", - "zcollection>=2026.6.0,<2027.0.0", "netcdf4>=1.6.5,<2.0.0", "tqdm>=4.66.4,<5.0.0", "pyogrio>=0.8.0,<0.9.0", - "pyarrow>=16.1.0,<17.0.0", + "pyarrow>=16.1.0,<19.0.0", "py-hydroweb>=1.2.0,<2.0.0", "folium>=0.16.0,<0.17.0", "branca>=0.7.2,<0.8.0", "xvec>=0.3.0,<0.4.0", "eodag>=4.4.0,<5.0.0", - "xdggs>=0.6.0,<0.7.0", + "xdggs>=0.6.0,<0.8.0", "scipy>=1.18.0,<2.0.0", "earthaccess>=0.19.0", + "zarr>=2.18,<3", + "dsitributed>=2026.8", ] [project.urls] From c5a6e368704831faa24a18c47c7113b8ce4bc3c1 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 14:36:04 +0100 Subject: [PATCH 10/13] [ENH] dependencies remove zcollection python 3.13 compatibility --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c4137cc..fa95eb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "scipy>=1.18.0,<2.0.0", "earthaccess>=0.19.0", "zarr>=2.18,<3", - "dsitributed>=2026.8", + "distributed>=2026.8", ] [project.urls] From 2553bd51a4bc1ce017bf73d8e35c5c1e035396fe Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 14:40:24 +0100 Subject: [PATCH 11/13] [ENH] dependencies remove zcollection python 3.13 compatibility --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fa95eb4..176a04a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "fiona>=1.9.5,<2.0.0", "pandas>=2.2.0,<3.0.0", "geopandas>=1.1.3,<2.0.0", - "h3>=3.7.6,<4.3.0", + "h3>=3.7.6,<4.0.0", "astropy>=8.0.0,<9.0.0", "astropy-healpix>=1.1.0,<2.0.0", "xarray>=2025.3.0,<2026.7.0", From 00a5b1dc9673598042b57bbd920e5a69793749ba Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 15:43:57 +0100 Subject: [PATCH 12/13] [ENH] dependencies remove zcollection python 3.13 compatibility --- .github/workflows/ci_notebook.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_notebook.yml b/.github/workflows/ci_notebook.yml index 052767b..6cd5dc5 100644 --- a/.github/workflows/ci_notebook.yml +++ b/.github/workflows/ci_notebook.yml @@ -37,7 +37,6 @@ jobs: papermill pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb /tmp/notebooks/download_pixc_to_gpkg papermill pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb /tmp/notebooks/download_pixc_to_zarr - papermill pixcdust/notebooks/convert/convert_to_zcollection.ipynb /tmp/notebooks/convert_to_zcollection papermill pixcdust/notebooks/convert/dggs_tuto.ipynb /tmp/notebooks/dggs_tuto papermill pixcdust/notebooks/read_and_use/read_netcdf.ipynb /tmp/notebooks/read_netcdf From 1057776d51898ccc08f23f2384233244ddd983d5 Mon Sep 17 00:00:00 2001 From: Cedric PENARD Date: Tue, 27 Oct 2026 17:45:57 +0100 Subject: [PATCH 13/13] [TEST] Fix pb in notebook --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 176a04a..f6113c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "branca>=0.7.2,<0.8.0", "xvec>=0.3.0,<0.4.0", "eodag>=4.4.0,<5.0.0", - "xdggs>=0.6.0,<0.8.0", + "xdggs>=0.6.0,<0.7.0", "scipy>=1.18.0,<2.0.0", "earthaccess>=0.19.0", "zarr>=2.18,<3",