diff --git a/src/fetchez/cli/pipeline.py b/src/fetchez/cli/pipeline.py index 7f691b4..3b96942 100644 --- a/src/fetchez/cli/pipeline.py +++ b/src/fetchez/cli/pipeline.py @@ -199,9 +199,9 @@ def format_commands(self, ctx, formatter): "--refresh", is_flag=True, help="Force fresh API fetch, bypassing local cache." ) @click.option( - "--ignore-failures", + "--fail-fast", is_flag=True, - help="Continue processing through failures (Warning: may result in incomplete data or products).", + help="Raise an exception on the first failure, otherwise continue processing through failures.", ) @click.pass_context # """Initializes the context before the chained subcommands run.""" @@ -216,7 +216,7 @@ def pipeline_group( threads, shared_cache, refresh, - ignore_failures, + fail_fast, ): """Fetch/download data and execute processing pipelines. @@ -269,7 +269,7 @@ def process_pipeline( threads, shared_cache, refresh, - ignore_failures, + fail_fast, ): """Executes after all chained commands have returned their dictionaries.""" @@ -324,5 +324,5 @@ def process_pipeline( else: click.secho("Executing dynamic pipeline...", fg="cyan", bold=True, err=True) Recipe.from_dict(config).run( - shared_cache=shared_cache, refresh=refresh, ignore_failures=ignore_failures + shared_cache=shared_cache, refresh=refresh, ignore_failures=not fail_fast ) diff --git a/src/fetchez/cli/recipes.py b/src/fetchez/cli/recipes.py index 5d683f6..419ffb6 100644 --- a/src/fetchez/cli/recipes.py +++ b/src/fetchez/cli/recipes.py @@ -309,9 +309,9 @@ def translate_recipe(name, as_json): "--refresh", is_flag=True, help="Force fresh API fetch, bypassing local cache." ) @click.option( - "--ignore-failures", + "--fail-fast", is_flag=True, - help="Continue processing through failures (Warning: may result in incomplete data or products).", + help="Raise an exception on the first failure, otherwise continue processing through failures.", ) @click.argument("name") def run_recipe( @@ -323,7 +323,7 @@ def run_recipe( modifier, schema, refresh, - ignore_failures, + fail_fast, ): """Execute a YAML recipe by registry name or file path.""" @@ -376,7 +376,7 @@ def run_recipe( outdir=outdir, shared_cache=shared_cache, refresh=refresh, - ignore_failures=ignore_failures, + ignore_failures=not fail_fast, ) click.secho(f"✨ Successfully executed {name} recipe!", fg="green", bold=True) diff --git a/src/fetchez/core.py b/src/fetchez/core.py index 628be61..57bcc7f 100644 --- a/src/fetchez/core.py +++ b/src/fetchez/core.py @@ -722,7 +722,7 @@ def fetch_file( if extras: status_msg += f" ({extras})" except Exception: - pass + req.raise_for_status() raise ConnectionError(status_msg) with open(part_fn, mode) as f: @@ -784,9 +784,9 @@ def fetch_file( ) time.sleep(wait_time) else: - logger.warning(f"Failed to download {self.url}: {e}") - # return -1 - req.raise_for_status() + logger.debug(f"Failed to download {self.url}: {e}") + return -1 + # req.raise_for_status() except filelock.Timeout: logger.error( @@ -878,7 +878,7 @@ def run_fetchez( modules: List[Any], threads: int = 3, global_hooks: Optional[List[Any]] = None, - ignore_failures: bool = False, + ignore_failures: bool = True, ): """Run Fetchez in parallel with hooks. diff --git a/src/fetchez/hooks/transfer_log.py b/src/fetchez/hooks/transfer_log.py index 36f997c..cda7710 100644 --- a/src/fetchez/hooks/transfer_log.py +++ b/src/fetchez/hooks/transfer_log.py @@ -40,7 +40,6 @@ def run(self, entries): for _mod, entry in entries: # Status 0 indicates success in fetchez - # Some modules don't return a status. if entry.get("status", 0) == 0: successes.append(entry) else: @@ -61,6 +60,10 @@ def run(self, entries): for entry in failures: f.write(f" [FAIL] {entry.get('url', 'Unknown URL')}\n") f.write(f" Target: {entry.get('dst_fn', 'Unknown')}\n") + f.write(f" Module: {entry.get('name', 'Unknown')}\n") + f.write( + f" Error: {entry.get('error_message', 'Unknown')}\n" + ) f.write("\n" + "=" * 80 + "\n") f.write(f"SUCCESSFUL DOWNLOADS ({len(successes)}):\n") diff --git a/src/fetchez/modules/base.py b/src/fetchez/modules/base.py index 2d9d8bd..fe5d134 100644 --- a/src/fetchez/modules/base.py +++ b/src/fetchez/modules/base.py @@ -365,6 +365,7 @@ def fetch_entry(self, entry, check_size=True, retries=5, verbose=True): except Exception as e: logger.error(f"Fetch failed for {entry['url']}: {e}") status = -1 + entry["status"] = status return status