diff --git a/alphapulldown/folding_backend/alphafold2_backend.py b/alphapulldown/folding_backend/alphafold2_backend.py index e4b6c498..307ca26c 100644 --- a/alphapulldown/folding_backend/alphafold2_backend.py +++ b/alphapulldown/folding_backend/alphafold2_backend.py @@ -25,6 +25,8 @@ #from alphapulldown.utils.calculate_rmsd import calculate_rmsd_and_superpose from alphapulldown.utils.modelling_setup import pad_input_features from alphapulldown.prediction import fast_kernels as fast_kernels_policy +from alphapulldown.prediction.inference_kernels import record_inference_kernels +from alphapulldown.utils.output_paths import debug_dir from alphapulldown.prediction.inference_flags import validate_model_configuration from alphapulldown.prediction.jax_compilation_cache import enable_persistent_compilation_cache from alphapulldown.prediction.resume import ( @@ -412,7 +414,8 @@ def _write_processed_msa_debug_artifact( ), query_sequence=_query_sequence_for_debug(multimeric_object), ) - out_path = os.path.join(output_dir, f"{model_name}_processed_msa.a3m") + os.makedirs(debug_dir(output_dir), exist_ok=True) + out_path = os.path.join(debug_dir(output_dir), f"{model_name}_processed_msa.a3m") with open(out_path, "wt") as handle: handle.write(a3m_text) logging.info(f"Wrote processed MSA A3M to {out_path}") @@ -497,8 +500,8 @@ def _write_processed_template_debug_artifacts( else: chain_index = np.asarray(_normalize_asym_id(processed_feature_dict)["asym_id"]).reshape(-1) - debug_dir = os.path.join(output_dir, "templates_debug") - os.makedirs(debug_dir, exist_ok=True) + templates_dir = os.path.join(debug_dir(output_dir), "templates") + os.makedirs(templates_dir, exist_ok=True) domain_names = processed_feature_dict.get("template_domain_names") if domain_names is None: @@ -522,14 +525,14 @@ def _write_processed_template_debug_artifacts( domain_name, f"template_{template_index}" ) debug_path = os.path.join( - debug_dir, f"{model_name}_{template_name}_idx{template_index}.pdb" + templates_dir, f"{model_name}_{template_name}_idx{template_index}.pdb" ) with open(debug_path, "wt") as handle: handle.write(protein.to_pdb(template_protein)) logging.info(f"Wrote processed template PDB to {debug_path}") except Exception as exc: error_path = os.path.join( - debug_dir, f"ERROR_{model_name}_template_{template_index}.txt" + templates_dir, f"ERROR_{model_name}_template_{template_index}.txt" ) with open(error_path, "wt") as handle: handle.write(f"Error: {exc}\n") @@ -575,11 +578,9 @@ def _msa_depths( def _record_inference_kernels(output_dir, model_name, model_runner) -> None: - """Record in inference_kernels.json whether a model ran ColabFold's fused kernels. + """Record whether a model ran ColabFold's fused kernels (``inference_kernels``). - Fast and stock predictions differ slightly in their numerics, so the record lets - ranking and analysis tell them apart. Read back from the runner's own config, so it - states what actually ran. + Read back from the runner's own config, so it states what actually ran. """ try: global_config = model_runner.config.model.global_config @@ -589,17 +590,7 @@ def _record_inference_kernels(output_dir, model_name, model_runner) -> None: "fused_kernels": bool(global_config.get("use_pallas", False)), "compute_capability": global_config.get("compute_capability", None), } - path = os.path.join(output_dir, "inference_kernels.json") - existing = {} - if os.path.exists(path): - try: - with open(path) as f: - existing = json.load(f) - except (OSError, ValueError): - existing = {} - existing[model_name] = record - with open(path, "w") as f: - json.dump(existing, f, indent=2, sort_keys=True) + record_inference_kernels(output_dir, model_name, record) class AlphaFold2Backend(FoldingBackend): diff --git a/alphapulldown/folding_backend/alphafold3_backend.py b/alphapulldown/folding_backend/alphafold3_backend.py index df3a8d6c..cf317a0a 100644 --- a/alphapulldown/folding_backend/alphafold3_backend.py +++ b/alphapulldown/folding_backend/alphafold3_backend.py @@ -45,6 +45,7 @@ from alphapulldown.prediction.inference_flags import validate_model_configuration from alphapulldown.prediction.jax_compilation_cache import enable_persistent_compilation_cache from alphapulldown.prediction.resume import finish_fold_record +from alphapulldown.utils.output_paths import debug_dir from alphapulldown.objects import MultimericObject, MonomericObject, ChoppedObject from alphapulldown.folding_backend.alphafold3_storage import post_prediction_process_af3 @@ -134,6 +135,7 @@ def prepare_input( features_directory=features_directory, debug_templates=debug_templates, debug_msas=debug_msas, + debug_dir=debug_dir(objects_to_model[0]['output_dir']) if objects_to_model else None, ) for entry in objects_to_model: object_to_model = entry['object'] diff --git a/alphapulldown/folding_backend/alphafold3_debug.py b/alphapulldown/folding_backend/alphafold3_debug.py index 911d2821..3beb1341 100644 --- a/alphapulldown/folding_backend/alphafold3_debug.py +++ b/alphapulldown/folding_backend/alphafold3_debug.py @@ -18,6 +18,7 @@ from alphapulldown.utils.af2_to_af3_msa import Af2ToAf3TranslationResult from alphapulldown.utils.msa_encoding import ids_to_a3m_af3 +from alphapulldown.utils.output_paths import debug_dir @dataclasses.dataclass(frozen=True, slots=True) @@ -43,6 +44,7 @@ def _write_translated_msa_debug_artifacts( ) -> None: if not chain_records or not translation_results: return + output_dir = debug_dir(output_dir) def _extract_paired_species_identifier(description: str) -> str: if not description.startswith(("tr|", "sp|")): @@ -229,11 +231,12 @@ def _write_final_msa_a3m( try: if output_dir is None: return - os.makedirs(output_dir, exist_ok=True) + output_dir = debug_dir(output_dir) def write_from_array(rows: np.ndarray, suffix: str): # AF3 uses a different integer alphabet for MSA arrays. a3m_text = ids_to_a3m_af3(rows) + os.makedirs(output_dir, exist_ok=True) a3m_path = os.path.join(output_dir, f"{job_name}_seed-{seed_value}_{suffix}.a3m") with open(a3m_path, 'wt') as f: f.write(a3m_text) diff --git a/alphapulldown/folding_backend/alphafold3_inputs.py b/alphapulldown/folding_backend/alphafold3_inputs.py index 35414916..a1f0eec9 100644 --- a/alphapulldown/folding_backend/alphafold3_inputs.py +++ b/alphapulldown/folding_backend/alphafold3_inputs.py @@ -644,10 +644,14 @@ def __init__( features_directory: str | None = None, debug_templates: bool | None = None, debug_msas: bool = False, + debug_dir: str | None = None, ) -> None: self.features_directory = features_directory self.debug_templates = debug_templates self.debug_msas = debug_msas + # The fold's debug/ directory: generated template mmCIFs, and a note on each + # template that failed to convert. Without it, no debug files are written. + self.debug_dir = debug_dir self.chains: list[ folding_input.ProteinChain | folding_input.RnaChain @@ -861,11 +865,12 @@ def _monomeric_to_chain( # Remove _entity_poly_seq to avoid mismatches during AF3 parsing new_mmcif_string = strip_entity_poly_seq_block(new_mmcif_string) # Optional debug dump of generated template mmCIF - if self.debug_templates: + if self.debug_templates and self.debug_dir is not None: try: - os.makedirs("templates_debug", exist_ok=True) + templates_dir = os.path.join(self.debug_dir, "templates") + os.makedirs(templates_dir, exist_ok=True) debug_fname = os.path.join( - "templates_debug", + templates_dir, f"generated_template_{pdb_code}_{chain_id_template}_chain{chain_id}_idx{i}.cif", ) with open(debug_fname, "w") as f_dbg: @@ -895,9 +900,12 @@ def _monomeric_to_chain( ) except Exception as e: logging.error(f"Error processing template {i} for chain {chain_id}: {e}") + if self.debug_dir is None: + raise try: - os.makedirs("templates_debug", exist_ok=True) - error_filename = os.path.join("templates_debug", f"ERROR_template_{i}_chain_{chain_id}.txt") + templates_dir = os.path.join(self.debug_dir, "templates") + os.makedirs(templates_dir, exist_ok=True) + error_filename = os.path.join(templates_dir, f"ERROR_template_{i}_chain_{chain_id}.txt") with open(error_filename, "w") as error_file: error_file.write(f"Error: {e}\n") error_file.write(f"Template index: {i}\n") diff --git a/alphapulldown/folding_backend/alphafold3_runner.py b/alphapulldown/folding_backend/alphafold3_runner.py index 306cbce4..bedd00c8 100644 --- a/alphapulldown/folding_backend/alphafold3_runner.py +++ b/alphapulldown/folding_backend/alphafold3_runner.py @@ -11,7 +11,6 @@ import dataclasses import functools import importlib -import json import logging import os import pathlib @@ -36,6 +35,7 @@ from alphapulldown.folding_backend.alphafold3_debug import _write_final_msa_a3m from alphapulldown.folding_backend.alphafold3_outputs import write_seed_outputs from alphapulldown.prediction import af3_fused_triangles +from alphapulldown.prediction.inference_kernels import record_inference_kernels # ----------------------------------------------------------------------------- @@ -191,13 +191,11 @@ def _record_inference_kernels( model_runner: ModelRunner, example: features.BatchDict, ) -> None: - """Record in inference_kernels.json which kernels a seed's triangle layers ran. + """Record which kernels a seed's triangle layers ran (``inference_kernels``). - Fused and original layers differ slightly in their numerics, so the record lets - ranking and analysis tell them apart, like AlphaFold 2's. Entries are keyed - ``seed-N`` like AF3's other outputs and are read back from the runner's own config, - so they state what actually ran. The record never fails a finished prediction: an - unreadable file is replaced, and any other failure is logged and skipped. + Entries are keyed ``seed-N`` like AF3's other outputs and are read back from the + runner's own config, so they state what actually ran. The record never fails a + finished prediction: a failure to describe the kernels is logged and skipped. """ fused_triangles = getattr(model_runner, "fused_triangles", None) tokens = example.get("token_index", example.get("aatype")) @@ -211,22 +209,7 @@ def _record_inference_kernels( except Exception as exc: # provenance only; the prediction itself succeeded logging.warning("Not recording the inference kernels of %s: %s", key, exc) return - path = pathlib.Path(output_dir) / "inference_kernels.json" - try: - records = json.loads(path.read_text()) if path.exists() else {} - if not isinstance(records, dict): - raise ValueError("not a JSON object") - except (OSError, ValueError) as exc: - logging.warning("Replacing unreadable %s: %s", path, exc) - records = {} - records[key] = record - try: - os.makedirs(output_dir, exist_ok=True) - temporary = path.with_suffix(".json.tmp") - temporary.write_text(json.dumps(records, indent=2, sort_keys=True) + "\n") - temporary.replace(path) - except OSError as exc: - logging.warning("Not recording the inference kernels of %s: %s", key, exc) + record_inference_kernels(output_dir, key, record) def predict_structure( diff --git a/alphapulldown/prediction/fold_preparation.py b/alphapulldown/prediction/fold_preparation.py index 9dfc6e44..e000ccbe 100644 --- a/alphapulldown/prediction/fold_preparation.py +++ b/alphapulldown/prediction/fold_preparation.py @@ -19,6 +19,7 @@ from absl import logging from alphapulldown.objects import ChoppedObject, MonomericObject, MultimericObject +from alphapulldown.utils.output_paths import FEATURE_METADATA_DIR # Most filesystems reject paths longer than this. @@ -94,14 +95,16 @@ def _copy_feature_metadata(interactors: List[Any], flags: Any, output_dir: str) ) continue latest = max(metadata_files, key=os.path.getmtime) - destination = os.path.join(output_dir, os.path.basename(latest)) + metadata_dir = os.path.join(output_dir, FEATURE_METADATA_DIR) + os.makedirs(metadata_dir, exist_ok=True) + destination = os.path.join(metadata_dir, os.path.basename(latest)) if latest.endswith(".json.xz"): destination = destination[: -len(".xz")] logging.info("Decompressing %s to %s", latest, destination) with lzma.open(latest, "rb") as source, open(destination, "wb") as target: target.write(source.read()) else: - logging.info("Copying %s to %s", latest, output_dir) + logging.info("Copying %s to %s", latest, metadata_dir) shutil.copyfile(latest, destination) diff --git a/alphapulldown/prediction/inference_kernels.py b/alphapulldown/prediction/inference_kernels.py new file mode 100644 index 00000000..3affb913 --- /dev/null +++ b/alphapulldown/prediction/inference_kernels.py @@ -0,0 +1,62 @@ +"""Which kernels predicted a fold: ``alphapulldown/inference_kernels.json`` in its directory. + +Fused and original kernels differ slightly in their numerics, so the record lets ranking +and analysis tell predictions apart. Each backend adds one entry per unit it predicts, +keyed ``model_`` by AlphaFold 2 and ``seed-`` by AlphaFold 3. Versions up to +2.10 kept the file at the top of the fold's directory; the next entry written moves it. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, Mapping + +from alphapulldown.utils.file_handling import open_atomic +from alphapulldown.utils.output_paths import alphapulldown_dir + +INFERENCE_KERNELS = "inference_kernels.json" + + +def inference_kernels_path(output_dir) -> str: + """Where the record of the fold in ``output_dir`` is kept.""" + return os.path.join(alphapulldown_dir(output_dir), INFERENCE_KERNELS) + + +def _legacy_path(output_dir) -> str: + return os.path.join(os.fspath(output_dir), INFERENCE_KERNELS) + + +def read_inference_kernels(output_dir) -> dict[str, Any]: + """The fold's record, wherever this version or an earlier one kept it; empty if + there is none or it is unreadable.""" + for path in (inference_kernels_path(output_dir), _legacy_path(output_dir)): + if not os.path.exists(path): + continue + try: + with open(path, encoding="utf-8") as handle: + records = json.load(handle) + if not isinstance(records, dict): + raise ValueError("not a JSON object") + except (OSError, ValueError) as exc: + logging.warning("Replacing unreadable %s: %s", path, exc) + return {} + return records + return {} + + +def record_inference_kernels(output_dir, key: str, entry: Mapping[str, Any]) -> None: + """Add ``entry`` under ``key``, keeping the other entries. Provenance never fails a + finished prediction: an unreadable record is replaced, a failed write only logged.""" + records = read_inference_kernels(output_dir) + records[key] = dict(entry) + try: + os.makedirs(alphapulldown_dir(output_dir), exist_ok=True) + with open_atomic(inference_kernels_path(output_dir)) as handle: + json.dump(records, handle, indent=2, sort_keys=True) + handle.write("\n") + if os.path.exists(_legacy_path(output_dir)): + os.remove(_legacy_path(output_dir)) + except OSError as exc: + logging.warning("Not recording the inference kernels of %s: %s", key, exc) diff --git a/alphapulldown/prediction/resume.py b/alphapulldown/prediction/resume.py index a2500c41..7739906c 100644 --- a/alphapulldown/prediction/resume.py +++ b/alphapulldown/prediction/resume.py @@ -1,7 +1,7 @@ """Writing prediction outputs so that an interrupted run can be resumed. Each fold directory holds a record of the run that predicted it -(``alphapulldown_prediction.json``): the settings that shape the fold's outputs, and +(``alphapulldown/prediction.json``): the settings that shape the fold's outputs, and whether the run finished. A backend starts the record before predicting and marks it complete as the last step of post-processing, so a later run with ``--allow_resume`` can skip a finished fold without inspecting, or redoing, any of its outputs. @@ -16,10 +16,16 @@ from absl import logging from alphapulldown.utils.file_handling import open_atomic +from alphapulldown.utils.output_paths import alphapulldown_dir PathLike = Union[str, "os.PathLike[str]"] -FOLD_RECORD_NAME = "alphapulldown_prediction.json" +FOLD_RECORD_NAME = "prediction.json" + + +def fold_record_path(output_dir: PathLike) -> str: + """Where the record of the fold in ``output_dir`` is kept.""" + return os.path.join(alphapulldown_dir(output_dir), FOLD_RECORD_NAME) def _as_recorded(settings: Mapping[str, Any]) -> Dict[str, Any]: @@ -29,7 +35,7 @@ def _as_recorded(settings: Mapping[str, Any]) -> Dict[str, Any]: def read_fold_record(output_dir: PathLike) -> Dict[str, Any]: """The record of the run that predicted the fold in ``output_dir``; empty if none.""" - path = os.path.join(os.fspath(output_dir), FOLD_RECORD_NAME) + path = fold_record_path(output_dir) try: with open(path, "r", encoding="utf-8") as handle: record = json.load(handle) @@ -44,8 +50,8 @@ def read_fold_record(output_dir: PathLike) -> Dict[str, Any]: def _write_fold_record(output_dir: PathLike, record: Mapping[str, Any]) -> None: from alphapulldown import __version__ - os.makedirs(output_dir, exist_ok=True) - with open_atomic(os.path.join(os.fspath(output_dir), FOLD_RECORD_NAME)) as handle: + os.makedirs(alphapulldown_dir(output_dir), exist_ok=True) + with open_atomic(fold_record_path(output_dir)) as handle: json.dump( {**record, "alphapulldown_version": __version__}, handle, indent=2, sort_keys=True ) diff --git a/alphapulldown/scripts/convert_to_modelcif.py b/alphapulldown/scripts/convert_to_modelcif.py index febc7784..7a9659f1 100755 --- a/alphapulldown/scripts/convert_to_modelcif.py +++ b/alphapulldown/scripts/convert_to_modelcif.py @@ -31,6 +31,7 @@ from alphapulldown.utils.file_handling import iter_seqs from alphapulldown.utils.feature_metadata import ( + copied_feature_metadata_paths, decode_metadata_from_description, read_feature_metadata, ) @@ -805,11 +806,7 @@ def _register_metadata(monomer_name: str, metadata: dict) -> None: if new_entry not in fasta_dicts: fasta_dicts.append(new_entry) - feature_json_files = sorted( - set(glob.glob(os.path.join(out_dir, "*_feature_metadata_*.json"))) - | set(glob.glob(os.path.join(out_dir, "*_feature_metadata_*.json.xz"))) - ) - for feature_json in feature_json_files: + for feature_json in copied_feature_metadata_paths(out_dir): _file_exists_or_exit( feature_json, f"No feature metadata file '{feature_json}' found." ) diff --git a/alphapulldown/scripts/run_structure_prediction.py b/alphapulldown/scripts/run_structure_prediction.py index ea0960d5..9f21a4dd 100644 --- a/alphapulldown/scripts/run_structure_prediction.py +++ b/alphapulldown/scripts/run_structure_prediction.py @@ -244,13 +244,15 @@ 'Folding backend: alphafold2, alphafold3, or alphalink. UniFold is unavailable.') flags.DEFINE_boolean( 'debug_templates', False, - 'If set, save backend-specific template debug artifacts. AF3 writes generated' - ' template mmCIFs and AF2 writes processed template PDBs.' + 'If set, save backend-specific template debug artifacts in each fold\'s' + ' debug/templates/ directory. AF3 writes generated template mmCIFs and AF2 writes' + ' processed template PDBs.' ) flags.DEFINE_boolean( 'debug_msas', False, - 'If set, dump backend-specific MSA debug artifacts such as translated or' - ' processed input A3Ms and the AF3 final merged complex A3M.' + 'If set, dump backend-specific MSA debug artifacts in each fold\'s debug/' + ' directory, such as translated or processed input A3Ms and the AF3 final merged' + ' complex A3M.' ) FLAGS = flags.FLAGS diff --git a/alphapulldown/utils/feature_metadata.py b/alphapulldown/utils/feature_metadata.py index 71b65793..3c8f74a5 100644 --- a/alphapulldown/utils/feature_metadata.py +++ b/alphapulldown/utils/feature_metadata.py @@ -13,7 +13,9 @@ import json import lzma from pathlib import Path -from typing import Any, Iterable, Mapping, Sequence +from typing import Any, Mapping, Sequence + +from alphapulldown.utils.output_paths import FEATURE_METADATA_DIR AF3_METADATA_MARKER = "\n__ALPHAPULLDOWN_FEATURE_METADATA_V1__=" @@ -168,13 +170,19 @@ def find_feature_metadata( return read_feature_metadata(newest) +def copied_feature_metadata_paths(directory: str | Path) -> list[Path]: + """The AF2 metadata sidecars copied into a prediction directory: in + ``alphapulldown/feature_metadata/``, or at its top where versions up to 2.10 put them.""" + root = Path(directory) + paths: set[Path] = set() + for location in (root / FEATURE_METADATA_DIR, root): + paths |= set(location.glob("*_feature_metadata_*.json")) + paths |= set(location.glob("*_feature_metadata_*.json.xz")) + return sorted(paths) + + def load_feature_metadata_sidecars( directory: str | Path, ) -> list[dict[str, Any]]: """Load all AF2 metadata sidecars copied into a prediction directory.""" - root = Path(directory) - paths: Iterable[Path] = sorted( - set(root.glob("*_feature_metadata_*.json")) - | set(root.glob("*_feature_metadata_*.json.xz")) - ) - return [read_feature_metadata(path) for path in paths] + return [read_feature_metadata(path) for path in copied_feature_metadata_paths(directory)] diff --git a/alphapulldown/utils/output_paths.py b/alphapulldown/utils/output_paths.py index 8e6b3b4a..a3dbd4c4 100644 --- a/alphapulldown/utils/output_paths.py +++ b/alphapulldown/utils/output_paths.py @@ -8,6 +8,24 @@ _AF3_ALLOWED_NAME_CHARS = set(string.ascii_lowercase + string.digits + "_-.") +# A fold's output directory holds AlphaFold's own outputs at the top; what AlphaPulldown +# adds for itself goes in two subdirectories. ``alphapulldown/`` holds the fold record +# (resume), the inference-kernel record and the copied feature metadata; ``debug/`` +# holds what the --debug_* flags write. +ALPHAPULLDOWN_DIR = "alphapulldown" +DEBUG_DIR = "debug" +FEATURE_METADATA_DIR = os.path.join(ALPHAPULLDOWN_DIR, "feature_metadata") + + +def alphapulldown_dir(output_dir) -> str: + """Where AlphaPulldown keeps its records of the fold in ``output_dir``.""" + return os.path.join(os.fspath(output_dir), ALPHAPULLDOWN_DIR) + + +def debug_dir(output_dir) -> str: + """Where the --debug_* flags write their files for the fold in ``output_dir``.""" + return os.path.join(os.fspath(output_dir), DEBUG_DIR) + def _raw_sanitise_af3_job_name(job_name: str) -> str: lower_spaceless_name = job_name.lower().replace(" ", "_") diff --git a/docs/af3_fused_triangles.md b/docs/af3_fused_triangles.md index f76d5fcc..922494fa 100644 --- a/docs/af3_fused_triangles.md +++ b/docs/af3_fused_triangles.md @@ -106,8 +106,9 @@ DeepMind's weights load unchanged. ## Output record -Every AlphaFold 3 output directory gets an `inference_kernels.json` with one entry per -seed, keyed like AlphaFold 3's other outputs: +Every AlphaFold 3 output directory gets an `alphapulldown/inference_kernels.json` with +one entry per seed, keyed like AlphaFold 3's other outputs (versions up to 2.10 wrote it at +the top of the directory): ```json { diff --git a/test/cluster/check_alphafold2_predictions.py b/test/cluster/check_alphafold2_predictions.py index 531f9351..57d19d62 100755 --- a/test/cluster/check_alphafold2_predictions.py +++ b/test/cluster/check_alphafold2_predictions.py @@ -509,17 +509,17 @@ def _assertSucceeded(self, res: subprocess.CompletedProcess, label: str) -> None ) def _assertComplete(self, fold_dir: Path) -> None: - record = json.loads((fold_dir / "alphapulldown_prediction.json").read_text()) + record = json.loads((fold_dir / "alphapulldown" / "prediction.json").read_text()) self.assertTrue(record["complete"], fold_dir) @staticmethod def _mtimes(*fold_dirs: Path) -> dict[str, int]: - """Each output's modification time; not the feature metadata, which fold - preparation copies in on every run before predicting.""" + """Each output's modification time, AlphaPulldown's records included; not the + feature metadata, which fold preparation copies in on every run before predicting.""" return { str(path): path.stat().st_mtime_ns for fold_dir in fold_dirs - for path in fold_dir.iterdir() + for path in fold_dir.rglob("*") if path.is_file() and "_feature_metadata_" not in path.name } @@ -796,7 +796,9 @@ def _predict_padded_batch(self, name: str, fold: str, *extra: str) -> Path: @staticmethod def _kernels(result_dir: Path) -> dict: - return json.loads((result_dir / "inference_kernels.json").read_text(encoding="utf-8")) + return json.loads( + (result_dir / "alphapulldown" / "inference_kernels.json").read_text(encoding="utf-8") + ) @staticmethod def _scores(result_dir: Path) -> dict: diff --git a/test/cluster/check_alphafold3_predictions.py b/test/cluster/check_alphafold3_predictions.py index 24c203b9..0f0031c2 100755 --- a/test/cluster/check_alphafold3_predictions.py +++ b/test/cluster/check_alphafold3_predictions.py @@ -1278,7 +1278,7 @@ def test_issue_588_precomputed_mmseqs_msas_preserve_af3_species_pairing(self): debug_msas=True, ) job_name = fold_input_obj.sanitised_name() - summary_path = self.output_dir / f"{job_name}_af2_to_af3_translation_summary.json" + summary_path = self.output_dir / "debug" / f"{job_name}_af2_to_af3_translation_summary.json" self.assertTrue(summary_path.is_file(), f"Missing translation summary {summary_path}") summary = json.loads(summary_path.read_text(encoding="utf-8")) @@ -1416,7 +1416,7 @@ def test_issue_588_mmseqs_af2_features_enable_af3_species_pairing_inference(self result_dir = self._resolve_single_af3_result_dir() summary_paths = sorted( - result_dir.glob("*_af2_to_af3_translation_summary.json") + (result_dir / "debug").glob("*_af2_to_af3_translation_summary.json") ) self.assertLen(summary_paths, 1) summary = json.loads(summary_paths[0].read_text(encoding="utf-8")) @@ -1472,7 +1472,7 @@ def test_issue_588_mmseqs_af2_features_enable_af3_species_pairing_trimer_inferen result_dir = self._resolve_single_af3_result_dir() summary_paths = sorted( - result_dir.glob("*_af2_to_af3_translation_summary.json") + (result_dir / "debug").glob("*_af2_to_af3_translation_summary.json") ) self.assertLen(summary_paths, 1) summary = json.loads(summary_paths[0].read_text(encoding="utf-8")) @@ -2582,7 +2582,7 @@ def _kernel_records(self, mode: str) -> dict[str, dict[str, Any]]: result_dir = self._resolve_single_af3_result_dir() with (result_dir / "ranking_scores.csv").open() as handle: seeds = {line.split(",")[0] for line in handle.readlines()[1:] if line.strip()} - records = json.loads((result_dir / "inference_kernels.json").read_text()) + records = json.loads((result_dir / "alphapulldown" / "inference_kernels.json").read_text()) self.assertEqual(set(records), {f"seed-{seed}" for seed in seeds}) self.assertLen(records, 2) for key, record in records.items(): @@ -2807,7 +2807,7 @@ class TestAlphaFold3Resume(_TestBase): FOLD = "A0A075B6L2:1" DIMER = "A0A075B6L2:2" - RECORD = "alphapulldown_prediction.json" + RECORD = Path("alphapulldown", "prediction.json") SKIPPED = "is already complete in" SAMPLES = 2 @@ -2843,9 +2843,9 @@ def _assertSucceeded(self, res: subprocess.CompletedProcess, label: str) -> None ) def _fold_dir(self, root: Path) -> Path: - records = sorted(root.rglob(self.RECORD)) + records = sorted(root.rglob(str(self.RECORD))) self.assertLen(records, 1, f"one fold record under {root}") - return records[0].parent + return records[0].parent.parent def _record(self, fold_dir: Path) -> dict: return json.loads((fold_dir / self.RECORD).read_text()) @@ -2857,7 +2857,7 @@ def _outputs(self, fold_dir: Path) -> dict[str, int]: str(path.relative_to(fold_dir)): path.stat().st_mtime_ns for path in fold_dir.rglob("*") if path.is_file() - and path.name != self.RECORD + and path.relative_to(fold_dir) != self.RECORD and "_feature_metadata_" not in path.name } @@ -2874,12 +2874,20 @@ def _leave_what_a_killed_run_leaves(self, fold_dir: Path) -> None: for path in list(fold_dir.iterdir()): if path.is_dir() and path.name.startswith("seed-"): shutil.rmtree(path) - elif path.name != self.RECORD and not path.name.endswith("_data.json"): + elif path.is_file() and not path.name.endswith("_data.json"): path.unlink() + # Its alphapulldown/ folder: the record and the copied feature metadata, no + # kernels record yet. + (fold_dir / "alphapulldown" / "inference_kernels.json").unlink(missing_ok=True) def _leave_what_an_older_version_leaves(self, fold_dir: Path) -> None: - """AlphaPulldown 2.10 and older wrote no fold record.""" - (fold_dir / self.RECORD).unlink() + """AlphaPulldown 2.10 and older wrote no fold record, and kept the kernels record + and the copied feature metadata at the top of the fold's directory.""" + own = fold_dir / "alphapulldown" + for path in [own / "inference_kernels.json", *own.glob("feature_metadata/*")]: + if path.is_file(): + path.rename(fold_dir / path.name) + shutil.rmtree(own) def _leave_what_a_run_killed_after_its_first_seed_leaves(self, fold_dir: Path) -> None: """Each seed is written out, and ranked, as it finishes; the best model only diff --git a/test/integration/test_af3_input_preparation.py b/test/integration/test_af3_input_preparation.py index 13bfa7d4..871a7c52 100644 --- a/test/integration/test_af3_input_preparation.py +++ b/test/integration/test_af3_input_preparation.py @@ -190,7 +190,7 @@ def test_issue_588_mmseqs_af2_features_produce_sane_af3_chain_input_msas(self): self.assertEqual(sorted(chain_sequences), ["A", "B"]) job_name = fold_input_obj.sanitised_name() - summary_path = self.output_dir / f"{job_name}_af2_to_af3_translation_summary.json" + summary_path = self.output_dir / "debug" / f"{job_name}_af2_to_af3_translation_summary.json" self.assertTrue(summary_path.is_file(), f"Missing translation summary {summary_path}") summary = json.loads(summary_path.read_text(encoding="utf-8")) @@ -230,7 +230,7 @@ def test_issue_588_mmseqs_af2_features_produce_sane_af3_chain_input_msas(self): ) for msa_kind in ("paired_input", "unpaired_input"): - msa_path = self.output_dir / f"{job_name}_chain-{chain_id}_{msa_kind}.a3m" + msa_path = self.output_dir / "debug" / f"{job_name}_chain-{chain_id}_{msa_kind}.a3m" self.assertTrue(msa_path.is_file(), f"Missing debug MSA {msa_path}") msa_text = msa_path.read_text(encoding="utf-8") if msa_text: @@ -408,6 +408,7 @@ def test_af3_real_fixture_pipeline_tolerates_mixed_missing_accession_ids(self): job_name = fold_input_obj.sanitised_name() summary_path = ( Path(prepared_output_dir) + / "debug" / f"{job_name}_af2_to_af3_translation_summary.json" ) self.assertTrue(summary_path.is_file(), f"Missing translation summary {summary_path}") diff --git a/test/unit/test_alphafold2_backend_helpers.py b/test/unit/test_alphafold2_backend_helpers.py index 892b2015..5faea6ea 100644 --- a/test/unit/test_alphafold2_backend_helpers.py +++ b/test/unit/test_alphafold2_backend_helpers.py @@ -459,7 +459,7 @@ def test_asym_query_and_msa_debug_helpers(af2_backend_module, tmp_path): model_name="modelA", ) - debug_path = tmp_path / "modelA_processed_msa.a3m" + debug_path = tmp_path / "debug" / "modelA_processed_msa.a3m" assert debug_path.read_text(encoding="utf-8") == ">query\nAC\n>rows\n1\n" @@ -511,7 +511,7 @@ def fake_protein_constructor(**kwargs): model_name="modelB", ) - debug_dir = tmp_path / "templates_debug" + debug_dir = tmp_path / "debug" / "templates" assert (debug_dir / "modelB_good_template_idx0.pdb").is_file() error_file = debug_dir / "ERROR_modelB_template_1.txt" assert error_file.is_file() @@ -1888,7 +1888,7 @@ def test_af2_rerun_of_a_finished_fold_whose_pickles_were_removed_predicts_nothin assert len(predictions) == 2 assert rerun == [] - record = json.loads((tmp_path / "alphapulldown_prediction.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "prediction.json").read_text()) assert record["complete"] is True assert record["settings"]["model_names"] == ["modelA", "modelB"] @@ -1911,7 +1911,7 @@ def test_af2_finished_fold_is_post_processed_again_when_its_relaxation_changes( assert len(predictions) == 2 assert len(rerun) == 1 assert len(list(tmp_path.glob("relaxed_*.pdb"))) == 1 - record = json.loads((tmp_path / "alphapulldown_prediction.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "prediction.json").read_text()) assert record["complete"] is True assert record["settings"]["models_to_relax"] == str( af2_backend_module.ModelsToRelax.BEST @@ -1933,12 +1933,12 @@ def test_af2_fold_killed_before_postprocessing_is_resumed( ) next(jobs) # predicted, then killed before post-processing - record = json.loads((tmp_path / "alphapulldown_prediction.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "prediction.json").read_text()) assert record["complete"] is False assert len(_af2_run(af2_backend_module, monkeypatch, tmp_path, predictions)) == 1 # Both models came from the first run's pickles. assert len(predictions) == 2 - record = json.loads((tmp_path / "alphapulldown_prediction.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "prediction.json").read_text()) assert record["complete"] is True @@ -2056,7 +2056,7 @@ def runner(use_pallas): # Not an AlphaFold RunModel: nothing is recorded, nothing fails. af2_backend_module._record_inference_kernels(tmp_path, "model_3", object()) - record = json.loads((tmp_path / "inference_kernels.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "inference_kernels.json").read_text()) assert record == { "model_1": {"compute_capability": 86, "fused_kernels": True}, "model_2": {"compute_capability": None, "fused_kernels": False}, diff --git a/test/unit/test_alphafold3_backend_helpers.py b/test/unit/test_alphafold3_backend_helpers.py index 056abd98..2ec39481 100644 --- a/test/unit/test_alphafold3_backend_helpers.py +++ b/test/unit/test_alphafold3_backend_helpers.py @@ -952,7 +952,7 @@ def extract_structures(self, batch, result, target_name): assert results[0].embeddings["single_embeddings"].shape == (2, 1) assert results[0].embeddings["pair_embeddings"].shape == (2, 2) assert results[0].distogram.shape == (2, 2) - final_msa = tmp_path / "job_name_seed-7_final_complex_msa.a3m" + final_msa = tmp_path / "debug" / "job_name_seed-7_final_complex_msa.a3m" assert final_msa.read_text(encoding="utf-8") == ">query\nABC\n" @@ -1011,14 +1011,15 @@ def test_predict_structure_records_kernels_per_seed_after_extraction( # After extraction, so it is not timed as model inference. assert events == ["extract", "record", "extract", "record"] - record = json.loads((tmp_path / "inference_kernels.json").read_text()) + record = json.loads((tmp_path / "alphapulldown" / "inference_kernels.json").read_text()) assert record == {"seed-7": OFF_RECORD, "seed-8": OFF_RECORD} def test_record_inference_kernels_replaces_a_corrupt_file( af3_runner_module, tmp_path, caplog ): - path = tmp_path / "inference_kernels.json" + path = tmp_path / "alphapulldown" / "inference_kernels.json" + path.parent.mkdir() path.write_text("{not json") af3_runner_module._record_inference_kernels( @@ -1030,7 +1031,8 @@ def test_record_inference_kernels_replaces_a_corrupt_file( def test_record_inference_kernels_keeps_other_entries(af3_runner_module, tmp_path): - path = tmp_path / "inference_kernels.json" + path = tmp_path / "alphapulldown" / "inference_kernels.json" + path.parent.mkdir() path.write_text(json.dumps({"seed-1": {"fused_kernels": True}})) af3_runner_module._record_inference_kernels( @@ -1055,7 +1057,7 @@ def broken(*args, **kwargs): tmp_path, 7, FakeKernelRunner(af3_runner_module), {"token_index": np.arange(3)} ) - assert not (tmp_path / "inference_kernels.json").exists() + assert not list(tmp_path.rglob("inference_kernels.json")) assert "no alphafold3.jax.fused_triangle" in caplog.text @@ -1066,7 +1068,7 @@ def test_record_inference_kernels_skips_runners_without_a_choice( tmp_path, 7, SimpleNamespace(), {"token_index": np.arange(3)} ) - assert not (tmp_path / "inference_kernels.json").exists() + assert not list(tmp_path.rglob("inference_kernels.json")) def test_af3_setup_builds_model_runner_and_validates_gpu_capability( @@ -1385,7 +1387,7 @@ def _af3_run(module, monkeypatch, output_dir, *, finish=True, random_seed=None, def _af3_fold_record(output_dir): - return json.loads((output_dir / "alphapulldown_prediction.json").read_text()) + return json.loads((output_dir / "alphapulldown" / "prediction.json").read_text()) def test_af3_rerun_of_a_finished_fold_predicts_nothing( @@ -2276,7 +2278,7 @@ def test_prepare_input_multimer_writes_translated_msa_debug_artifacts( ) assert fold_input.chains[1].unpaired_msa == ">unpairedB\nFGHI\n" - summary_path = tmp_path / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" + summary_path = tmp_path / "debug" / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" assert summary_path.is_file() summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["translation_modes"] == [ @@ -2313,14 +2315,14 @@ def test_prepare_input_multimer_writes_translated_msa_debug_artifacts( }, ] assert ( - tmp_path / f"{fold_input.sanitised_name()}_chain-A_paired_input.a3m" + tmp_path / "debug" / f"{fold_input.sanitised_name()}_chain-A_paired_input.a3m" ).read_text(encoding="utf-8") == ( ">query\nACDE\n" ">tr|APA0000001|APA0000001_ECOLX\nACDE\n" ">tr|APA0000002|APA0000002_SHIDY\nACDE\n" ) assert ( - tmp_path / f"{fold_input.sanitised_name()}_chain-B_unpaired_input.a3m" + tmp_path / "debug" / f"{fold_input.sanitised_name()}_chain-B_unpaired_input.a3m" ).read_text(encoding="utf-8") == ">unpairedB\nFGHI\n" @@ -2554,7 +2556,7 @@ def _unexpected_fallback(**kwargs): assert fold_input.chains[1].paired_msa == "" assert fold_input.chains[2].paired_msa - summary_path = tmp_path / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" + summary_path = tmp_path / "debug" / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["translation_modes"] == ["af3_species_pairing_from_af2_individual_msas"] assert summary["translated_paired_input_row_count"] == 2 @@ -2630,7 +2632,7 @@ def _unexpected_fallback(**kwargs): assert resolve_msa_overlaps is False assert all(chain.paired_msa for chain in fold_input.chains) - summary_path = tmp_path / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" + summary_path = tmp_path / "debug" / f"{fold_input.sanitised_name()}_af2_to_af3_translation_summary.json" summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["translation_modes"] == ["af3_species_pairing_from_af2_individual_msas"] assert summary["translated_paired_input_row_count"] == 3 @@ -2718,7 +2720,8 @@ def _fake_to_mmcif(*, prot, file_id, model_type): assert "_pdbx_audit_revision_history.revision_date" in chain.templates[0].mmcif debug_path = ( tmp_path - / "templates_debug" + / "debug" + / "templates" / "generated_template_1abc_B_chainA_idx1.cif" ) assert debug_path.read_text(encoding="utf-8") == chain.templates[0].mmcif @@ -2755,20 +2758,24 @@ def test_prepare_input_writes_template_error_artifacts_on_mmcif_generation_failu "to_mmcif", lambda **kwargs: (_ for _ in ()).throw(RuntimeError("mmcif boom")), ) - monkeypatch.chdir(tmp_path) + workdir = tmp_path / "workdir" + workdir.mkdir() + monkeypatch.chdir(workdir) with pytest.raises(RuntimeError, match="mmcif boom"): af3_backend_module.AlphaFold3Backend.prepare_input( objects_to_model=[ { "object": monomer, - "output_dir": str(tmp_path), + "output_dir": str(tmp_path / "fold"), } ], random_seed=41, ) - error_path = tmp_path / "templates_debug" / "ERROR_template_0_chain_A.txt" + # Beside the fold, not in whatever directory the job happens to run in. + assert list(workdir.iterdir()) == [] + error_path = tmp_path / "fold" / "debug" / "templates" / "ERROR_template_0_chain_A.txt" error_report = error_path.read_text(encoding="utf-8") assert "Error: mmcif boom" in error_report assert "Template index: 0" in error_report @@ -2906,7 +2913,8 @@ def model_params(self): def test_record_inference_kernels_replaces_a_file_that_is_not_an_object( af3_runner_module, tmp_path, caplog ): - path = tmp_path / "inference_kernels.json" + path = tmp_path / "alphapulldown" / "inference_kernels.json" + path.parent.mkdir() path.write_text("[1, 2]") af3_runner_module._record_inference_kernels( @@ -2980,11 +2988,11 @@ def test_translated_msa_summary_pairs_only_rows_with_a_species(af3_debug_module, translation_results=[translation], ) - summary = json.loads((tmp_path / "job_af2_to_af3_translation_summary.json").read_text()) + summary = json.loads((tmp_path / "debug" / "job_af2_to_af3_translation_summary.json").read_text()) # Only the HUMAN row pairs across the chains; the row without a species does not. assert summary["effective_paired_row_count"] == 1 assert summary["effective_paired_row_histogram_by_num_chains"] == {"2": 1} - assert (tmp_path / "job_chain-B_paired_input.a3m").read_text() == paired + assert (tmp_path / "debug" / "job_chain-B_paired_input.a3m").read_text() == paired @pytest.mark.parametrize( diff --git a/test/unit/test_feature_metadata.py b/test/unit/test_feature_metadata.py index 8a72a2de..abf9a48d 100644 --- a/test/unit/test_feature_metadata.py +++ b/test/unit/test_feature_metadata.py @@ -14,6 +14,7 @@ encode_metadata_in_description, extract_metadata_from_af3_json, extract_metadata_from_fold_input, + copied_feature_metadata_paths, find_feature_metadata, load_feature_metadata_sidecars, ) @@ -164,3 +165,17 @@ def test_embed_rejects_non_af3_or_ligand_only_payloads(): {"sequences": [{"ligand": {"id": "L", "ccdCodes": ["ATP"]}}]}, METADATA, ) + + +def test_copied_feature_metadata_is_read_from_its_folder_and_from_older_outputs(tmp_path): + # This version copies sidecars into alphapulldown/feature_metadata/; versions up to + # 2.10 put them at the top of the prediction directory, still read for ModelCIF. + new = tmp_path / "alphapulldown" / "feature_metadata" / "A_feature_metadata_2026-10-14.json" + old = tmp_path / "B_feature_metadata_2025-07-01.json.xz" + new.parent.mkdir(parents=True) + new.write_text(json.dumps(METADATA), encoding="utf-8") + old.write_bytes(lzma.compress(json.dumps(METADATA).encode("utf-8"))) + (tmp_path / "ranking_debug.json").write_text("{}", encoding="utf-8") + + assert copied_feature_metadata_paths(tmp_path) == sorted([new, old]) + assert load_feature_metadata_sidecars(tmp_path) == [METADATA, METADATA] diff --git a/test/unit/test_inference_kernels_record.py b/test/unit/test_inference_kernels_record.py new file mode 100644 index 00000000..392b91bb --- /dev/null +++ b/test/unit/test_inference_kernels_record.py @@ -0,0 +1,48 @@ +"""The record of which kernels predicted a fold, in its alphapulldown/ folder.""" + +import json + +from alphapulldown.prediction import inference_kernels + + +def _record(output_dir): + return json.loads((output_dir / "alphapulldown" / "inference_kernels.json").read_text()) + + +def test_entries_accumulate_in_the_alphapulldown_folder(tmp_path): + inference_kernels.record_inference_kernels(tmp_path, "seed-1", {"fused": True}) + inference_kernels.record_inference_kernels(tmp_path, "seed-2", {"fused": False}) + + assert _record(tmp_path) == {"seed-1": {"fused": True}, "seed-2": {"fused": False}} + assert sorted(path.name for path in tmp_path.iterdir()) == ["alphapulldown"] + + +def test_a_record_left_by_2_10_is_moved_into_the_folder(tmp_path): + # Resuming a fold predicted by 2.10, which kept the record at the top. + (tmp_path / "inference_kernels.json").write_text(json.dumps({"model_1": {"fused": False}})) + + assert inference_kernels.read_inference_kernels(tmp_path) == {"model_1": {"fused": False}} + inference_kernels.record_inference_kernels(tmp_path, "model_2", {"fused": True}) + + assert _record(tmp_path) == {"model_1": {"fused": False}, "model_2": {"fused": True}} + assert not (tmp_path / "inference_kernels.json").exists() + + +def test_an_unreadable_record_is_replaced(tmp_path, caplog): + path = tmp_path / "alphapulldown" / "inference_kernels.json" + path.parent.mkdir() + path.write_text("{truncated") + + inference_kernels.record_inference_kernels(tmp_path, "seed-1", {"fused": True}) + + assert _record(tmp_path) == {"seed-1": {"fused": True}} + assert "Replacing unreadable" in caplog.text + + +def test_a_record_that_cannot_be_written_never_fails_the_prediction(tmp_path, caplog): + # alphapulldown/ is a file, so the folder cannot be made. + (tmp_path / "alphapulldown").write_text("in the way") + + inference_kernels.record_inference_kernels(tmp_path, "seed-1", {"fused": True}) + + assert "Not recording the inference kernels of seed-1" in caplog.text diff --git a/test/unit/test_prediction_batch.py b/test/unit/test_prediction_batch.py index a2178ff3..638aca41 100644 --- a/test/unit/test_prediction_batch.py +++ b/test/unit/test_prediction_batch.py @@ -699,7 +699,7 @@ def test_metadata_is_taken_from_every_feature_directory(tmp_path): flags = SimpleNamespace(features_directory=[str(first), str(second)]) fold_preparation._copy_feature_metadata([interactor], flags, str(out)) - copied = sorted(p.name for p in out.iterdir()) + copied = sorted(p.name for p in (out / "alphapulldown" / "feature_metadata").iterdir()) assert copied == ["P1_feature_metadata_2025-01-01.json"] diff --git a/test/unit/test_prediction_resume.py b/test/unit/test_prediction_resume.py index 2ab60ba9..46c71252 100644 --- a/test/unit/test_prediction_resume.py +++ b/test/unit/test_prediction_resume.py @@ -43,6 +43,9 @@ def test_finishing_a_fold_without_a_record_writes_none(tmp_path): def test_an_unreadable_fold_record_counts_as_none(tmp_path): from alphapulldown.prediction import resume - (tmp_path / resume.FOLD_RECORD_NAME).write_text('{"complete": tr', encoding="utf-8") + path = tmp_path / "alphapulldown" / "prediction.json" + assert resume.fold_record_path(tmp_path) == str(path) + path.parent.mkdir() + path.write_text('{"complete": tr', encoding="utf-8") assert resume.read_fold_record(tmp_path) == {} diff --git a/test/unit/test_script_entrypoints.py b/test/unit/test_script_entrypoints.py index cdf81875..cd3c0d75 100644 --- a/test/unit/test_script_entrypoints.py +++ b/test/unit/test_script_entrypoints.py @@ -249,6 +249,7 @@ def __init__( modelling_setup_mod.parse_fold = lambda inputs, features_directory, delimiter: [] output_paths_mod = types.ModuleType("alphapulldown.utils.output_paths") + output_paths_mod.FEATURE_METADATA_DIR = os.path.join("alphapulldown", "feature_metadata") output_paths_mod.resolve_af3_combined_json_output_dir = ( lambda json_inputs, out_dir, use_ap_style: out_dir ) @@ -679,7 +680,10 @@ def test_pre_modelling_setup_decompresses_metadata_and_sets_input_sequences( assert returned_object is monomer assert returned_object.input_seqs == ["ACDE"] - copied_metadata = Path(returned_output_dir) / "protA_feature_metadata_2026-03-30.json" + copied_metadata = ( + Path(returned_output_dir) / "alphapulldown" / "feature_metadata" + / "protA_feature_metadata_2026-03-30.json" + ) assert copied_metadata.read_text(encoding="utf-8") == '{"meta": 1}'