diff --git a/src/fetchez/modules/base.py b/src/fetchez/modules/base.py index fe5d134..9141985 100644 --- a/src/fetchez/modules/base.py +++ b/src/fetchez/modules/base.py @@ -386,6 +386,24 @@ def add_entry_to_results(self, url: str, dst_fn: str, data_type: Any, **kwargs): if hasattr(self, "stream_kwargs"): entry.update(self.stream_kwargs) + # Intercept standard spatial metadata fields + standard_metadata = { + "name": self.name, # Module name (e.g., 'tnm', 'copernicus') + "title": kwargs.pop("title", "Unknown"), + "source": getattr(self, "meta_agency", "Unknown"), + "date": kwargs.pop("date", "Unknown"), + "data_type": data_type, + "resolution": kwargs.pop( + "resolution", getattr(self, "meta_resolution", "Unknown") + ), + "hdatum": kwargs.pop("hdatum", "Unknown"), + "vdatum": kwargs.pop("vdatum", "Unknown"), + "url": str(url), + } + entry_metadata = kwargs.pop("metadata", {}) + standard_metadata.update(entry_metadata) + entry["metadata"] = standard_metadata + entry.update(kwargs) self.results.append(entry) diff --git a/src/fetchez/modules/tnm.py b/src/fetchez/modules/tnm.py index f081466..8749b98 100644 --- a/src/fetchez/modules/tnm.py +++ b/src/fetchez/modules/tnm.py @@ -222,8 +222,26 @@ def run(self): item_bbox.get("maxY"), ) - # Extract the tile footprint (e.g., 'n35w120') - fn_bn = filename.split("_")[-2] + # Extract the tile footprint or project ID based on dataset type + if ( + "ned19" in filename.lower() + or "opr" in filename.lower() + or "lpc" in filename.lower() + ): + fn_bn = "_".join(filename.split("_")[:-1]) + elif bounds: + fn_bn = f"{round(bounds[0], 4)}_{round(bounds[1], 4)}_{round(bounds[2], 4)}_{round(bounds[3], 4)}" + else: + fn_bn = item.get("title", filename) + + # date = item.get("publicationDate", "") + # if bounds: + # bounds_str = f"{round(bounds[0], 4)}_{round(bounds[1], 4)}_{round(bounds[2], 4)}_{round(bounds[3], 4)}" + # project_id = "/".join(item.get("title", "")) + # fn_bn = f"{bounds_str}_{project_id}" + # else: + # fn_bn = item.get("title", filename) + date = item.get("publicationDate", "") item_data = { diff --git a/src/fetchez/recipes/modifiers/exclude_module.py b/src/fetchez/recipes/modifiers/exclude_module.py index bd36c51..03b6e7d 100644 --- a/src/fetchez/recipes/modifiers/exclude_module.py +++ b/src/fetchez/recipes/modifiers/exclude_module.py @@ -26,7 +26,7 @@ class ExcludeModuleModifier(BaseModifier): """ name = "exclude_module" - meta_desc = "Exclude specific modules from a recipe by name" + meta_desc = "Exclude specific modules from a recipe by name." def __init__(self, modules=None, **kwargs): super().__init__(**kwargs) diff --git a/src/fetchez/recipes/modifiers/inject_args.py b/src/fetchez/recipes/modifiers/inject_args.py new file mode 100644 index 0000000..082040f --- /dev/null +++ b/src/fetchez/recipes/modifiers/inject_args.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +fetchez.recipes.modifiers.inject_args +~~~~~~~~~~~~~~~~ + +Recipe mutator to remove specifically named modules +from a recipe. + +:copyright: (c) 2012 - 2026 CIRES Coastal DEM Team +:license: MIT, see LICENSE for more details. +""" + +from fetchez.recipes.modifiers.base import BaseModifier +import logging + +logger = logging.getLogger(__name__) + + +class InjectArgsModifier(BaseModifier): + """Injects arbitrary key:value arguments into matching modules or hooks. + Example: --modifier inject_args:match=stream_reproject,cache_dir=socal_data + """ + + name = "inject_args" + meta_desc = "Injects arbitrary key:value arguments into matching modules or hooks." + + def __init__(self, match=None, **kwargs): + super().__init__(**kwargs) + self.match = match + self.inject_kwargs = kwargs + + def _inject_into_item(self, item, match_name): + """Helper to safely mutate string or dict-based definitions.""" + + if isinstance(item, str) and item == match_name: + if self.inject_kwargs: + logger.info( + f"Modifier '{self.name}': Upgrading '{item}' to inject args." + ) + return {item: self.inject_kwargs.copy()} + return item + + if isinstance(item, dict): + key = list(item.keys())[0] + if key == match_name: + val = item[key] + if val is None: + val = {} + elif isinstance(val, str): + val = {"_value": val} + + if isinstance(val, dict): + val.update(self.inject_kwargs) + logger.info( + f"Modifier '{self.name}': Injected {list(self.inject_kwargs.keys())} into '{key}'." + ) + item[key] = val + + return item + + def apply(self, config): + """Mutates the recipe config by injecting arguments into matches.""" + + if not self.match or not self.inject_kwargs: + logger.warning( + "InjectArgsModifier requires a 'match' target and arguments to inject. Skipping." + ) + return config + + if "hooks" in config: + config["hooks"] = [ + self._inject_into_item(h, self.match) for h in config["hooks"] + ] + + if "modules" in config: + updated_modules = [] + for mod in config["modules"]: + mod = self._inject_into_item(mod, self.match) + + if isinstance(mod, dict): + mod_name = list(mod.keys())[0] + mod_args = mod[mod_name] + if isinstance(mod_args, dict) and "hooks" in mod_args: + mod_args["hooks"] = [ + self._inject_into_item(h, self.match) + for h in mod_args["hooks"] + ] + + updated_modules.append(mod) + config["modules"] = updated_modules + + return config