From a55e6ed9b1d33696839cba90665eaa587f53010f Mon Sep 17 00:00:00 2001 From: Matthew Love Date: Mon, 3 Aug 2026 10:22:34 -0700 Subject: [PATCH 1/5] fix typo in docs --- README.md | 2 +- docs/source/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9afbf52..fea615b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Project Chat

-**Fetchez** is a robust, highly modular and extensible Python framework designed to orchestrate complex geospatial data engineering workflows. +**Fetchez** is a robust, highly modular and extendable Python framework designed to orchestrate complex geospatial data engineering workflows. Originally developed as the core fetching engine for the [CUDEM](https://github.com/ciresdem/cudem) project, Fetchez has evolved into a standalone geospatial ETL platform. It seamlessly retrieves Bathymetry, Topography, Imagery, and Oceanographic data from dozens of global repositories (NOAA, USGS, Copernicus, ESA) and processes it on the fly. diff --git a/docs/source/index.md b/docs/source/index.md index f869047..07db650 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -4,7 +4,7 @@ *Fetchez Les Données* -**Fetchez** is a robust, highly modular and extensible Python framework designed to orchestrate complex geospatial data engineering workflows. +**Fetchez** is a robust, highly modular and extendable Python framework designed to orchestrate complex geospatial data engineering workflows. Originally developed as the core fetching engine for the [CUDEM](https://github.com/continuous-dems/cudem) project, Fetchez has evolved into a standalone ETL platform. It seamlessly retrieves Bathymetry, Topography, Imagery, and Oceanographic data from dozens of global repositories (NOAA, USGS, Copernicus, ESA) and processes it on the fly. From e6ace8b9316dc8cb1981778c7c9a785b38d030e9 Mon Sep 17 00:00:00 2001 From: Matthew Love Date: Mon, 3 Aug 2026 10:24:56 -0700 Subject: [PATCH 2/5] fix typo in docs --- README.md | 2 +- docs/source/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fea615b..9afbf52 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Project Chat

-**Fetchez** is a robust, highly modular and extendable Python framework designed to orchestrate complex geospatial data engineering workflows. +**Fetchez** is a robust, highly modular and extensible Python framework designed to orchestrate complex geospatial data engineering workflows. Originally developed as the core fetching engine for the [CUDEM](https://github.com/ciresdem/cudem) project, Fetchez has evolved into a standalone geospatial ETL platform. It seamlessly retrieves Bathymetry, Topography, Imagery, and Oceanographic data from dozens of global repositories (NOAA, USGS, Copernicus, ESA) and processes it on the fly. diff --git a/docs/source/index.md b/docs/source/index.md index 07db650..f869047 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -4,7 +4,7 @@ *Fetchez Les Données* -**Fetchez** is a robust, highly modular and extendable Python framework designed to orchestrate complex geospatial data engineering workflows. +**Fetchez** is a robust, highly modular and extensible Python framework designed to orchestrate complex geospatial data engineering workflows. Originally developed as the core fetching engine for the [CUDEM](https://github.com/continuous-dems/cudem) project, Fetchez has evolved into a standalone ETL platform. It seamlessly retrieves Bathymetry, Topography, Imagery, and Oceanographic data from dozens of global repositories (NOAA, USGS, Copernicus, ESA) and processes it on the fly. From f7149d85925a6a32d0dcdd7ebdfae8cb82359fd6 Mon Sep 17 00:00:00 2001 From: Matthew Love Date: Mon, 3 Aug 2026 12:56:39 -0700 Subject: [PATCH 3/5] fail-fast instead of ignore-failures --- src/fetchez/cli/pipeline.py | 10 +++++----- src/fetchez/cli/recipes.py | 8 ++++---- src/fetchez/core.py | 8 ++++---- src/fetchez/modules/base.py | 1 + 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/fetchez/cli/pipeline.py b/src/fetchez/cli/pipeline.py index 7f691b4..3b96942 100644 --- a/src/fetchez/cli/pipeline.py +++ b/src/fetchez/cli/pipeline.py @@ -199,9 +199,9 @@ def format_commands(self, ctx, formatter): "--refresh", is_flag=True, help="Force fresh API fetch, bypassing local cache." ) @click.option( - "--ignore-failures", + "--fail-fast", is_flag=True, - help="Continue processing through failures (Warning: may result in incomplete data or products).", + help="Raise an exception on the first failure, otherwise continue processing through failures.", ) @click.pass_context # """Initializes the context before the chained subcommands run.""" @@ -216,7 +216,7 @@ def pipeline_group( threads, shared_cache, refresh, - ignore_failures, + fail_fast, ): """Fetch/download data and execute processing pipelines. @@ -269,7 +269,7 @@ def process_pipeline( threads, shared_cache, refresh, - ignore_failures, + fail_fast, ): """Executes after all chained commands have returned their dictionaries.""" @@ -324,5 +324,5 @@ def process_pipeline( else: click.secho("Executing dynamic pipeline...", fg="cyan", bold=True, err=True) Recipe.from_dict(config).run( - shared_cache=shared_cache, refresh=refresh, ignore_failures=ignore_failures + shared_cache=shared_cache, refresh=refresh, ignore_failures=not fail_fast ) diff --git a/src/fetchez/cli/recipes.py b/src/fetchez/cli/recipes.py index 5d683f6..419ffb6 100644 --- a/src/fetchez/cli/recipes.py +++ b/src/fetchez/cli/recipes.py @@ -309,9 +309,9 @@ def translate_recipe(name, as_json): "--refresh", is_flag=True, help="Force fresh API fetch, bypassing local cache." ) @click.option( - "--ignore-failures", + "--fail-fast", is_flag=True, - help="Continue processing through failures (Warning: may result in incomplete data or products).", + help="Raise an exception on the first failure, otherwise continue processing through failures.", ) @click.argument("name") def run_recipe( @@ -323,7 +323,7 @@ def run_recipe( modifier, schema, refresh, - ignore_failures, + fail_fast, ): """Execute a YAML recipe by registry name or file path.""" @@ -376,7 +376,7 @@ def run_recipe( outdir=outdir, shared_cache=shared_cache, refresh=refresh, - ignore_failures=ignore_failures, + ignore_failures=not fail_fast, ) click.secho(f"✨ Successfully executed {name} recipe!", fg="green", bold=True) diff --git a/src/fetchez/core.py b/src/fetchez/core.py index 628be61..4d61824 100644 --- a/src/fetchez/core.py +++ b/src/fetchez/core.py @@ -722,7 +722,7 @@ def fetch_file( if extras: status_msg += f" ({extras})" except Exception: - pass + req.raise_for_status() raise ConnectionError(status_msg) with open(part_fn, mode) as f: @@ -784,9 +784,9 @@ def fetch_file( ) time.sleep(wait_time) else: - logger.warning(f"Failed to download {self.url}: {e}") - # return -1 - req.raise_for_status() + logger.debug(f"Failed to download {self.url}: {e}") + return -1 + # req.raise_for_status() except filelock.Timeout: logger.error( diff --git a/src/fetchez/modules/base.py b/src/fetchez/modules/base.py index 2d9d8bd..fe5d134 100644 --- a/src/fetchez/modules/base.py +++ b/src/fetchez/modules/base.py @@ -365,6 +365,7 @@ def fetch_entry(self, entry, check_size=True, retries=5, verbose=True): except Exception as e: logger.error(f"Fetch failed for {entry['url']}: {e}") status = -1 + entry["status"] = status return status From 1e68422d039e1345077ea0105132587c220889bc Mon Sep 17 00:00:00 2001 From: Matthew Love Date: Mon, 3 Aug 2026 13:04:02 -0700 Subject: [PATCH 4/5] transfer-log hook better reporting --- src/fetchez/hooks/transfer_log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fetchez/hooks/transfer_log.py b/src/fetchez/hooks/transfer_log.py index 36f997c..cda7710 100644 --- a/src/fetchez/hooks/transfer_log.py +++ b/src/fetchez/hooks/transfer_log.py @@ -40,7 +40,6 @@ def run(self, entries): for _mod, entry in entries: # Status 0 indicates success in fetchez - # Some modules don't return a status. if entry.get("status", 0) == 0: successes.append(entry) else: @@ -61,6 +60,10 @@ def run(self, entries): for entry in failures: f.write(f" [FAIL] {entry.get('url', 'Unknown URL')}\n") f.write(f" Target: {entry.get('dst_fn', 'Unknown')}\n") + f.write(f" Module: {entry.get('name', 'Unknown')}\n") + f.write( + f" Error: {entry.get('error_message', 'Unknown')}\n" + ) f.write("\n" + "=" * 80 + "\n") f.write(f"SUCCESSFUL DOWNLOADS ({len(successes)}):\n") From f81ee5f1f554e327e04b7d5e317c5a39d0be58d5 Mon Sep 17 00:00:00 2001 From: Matthew Love Date: Mon, 3 Aug 2026 13:09:22 -0700 Subject: [PATCH 5/5] ignore_failures is true by default --- src/fetchez/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fetchez/core.py b/src/fetchez/core.py index 4d61824..57bcc7f 100644 --- a/src/fetchez/core.py +++ b/src/fetchez/core.py @@ -878,7 +878,7 @@ def run_fetchez( modules: List[Any], threads: int = 3, global_hooks: Optional[List[Any]] = None, - ignore_failures: bool = False, + ignore_failures: bool = True, ): """Run Fetchez in parallel with hooks.