Add Regression Tests for File Versions

.gitignore

@@ -7,6 +7,7 @@ __pycache__
 /large_files/
 *.h5
 !/docs/data/*.h5
+!/tests/test_data/file_versions/*.h5
 venv
 profile*
 *-checkpoint.ipynb

tests/generate_file_version_data.py

@@ -0,0 +1,117 @@
+import argparse
+import json
+import random
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+import h5py
+import numpy as np
+
+from paretobench import Experiment, History
+from utils import experiment_to_manifest
+
+DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "test_data" / "file_versions"
+FILENAME_FMT = "paretobench_file_format_v{version}.h5"
+
+
+def make_experiment():
+    """
+    Build the synthetic experiment which gets saved into the fixtures. The contents are deterministic so that
+    running this script twice with the same version of the library produces the same file. Names and the
+    objective / constraint settings are included so that every serialized field is covered.
+
+    Returns
+    -------
+    Experiment
+        The experiment to save
+    """
+    random.seed(0)
+    np.random.seed(0)
+
+    runs = []
+    for problem in ["ZDT1 (n=4)", "CTP1 (n=4)", "TNK"]:
+        run = History.from_random(
+            n_populations=4,
+            n_objectives=3,
+            n_decision_vars=4,
+            n_constraints=2,
+            pop_size=25,
+            generate_names=True,
+            generate_obj_constraint_settings=True,
+        )
+        run.problem = problem
+        runs.append(run)
+
+    return Experiment(
+        runs=runs,
+        name="file_format_regression",
+        author="ParetoBench test suite",
+        software="ParetoBench",
+        software_version="0.0.0",
+        comment="Synthetic data used by the file format regression tests",
+        creation_time=datetime(2025, 1, 1, tzinfo=timezone.utc),
+    )
+
+
+def write_manifest(path):
+    """
+    Record what a saved file contains in a JSON file next to it. The description is taken from the file as it
+    reads back off of disk so that it captures the output of the reader and not the object which was saved.
+
+    Parameters
+    ----------
+    path : Path
+        The saved HDF5 file to describe
+
+    Returns
+    -------
+    Path
+        Path of the manifest which was written
+    """
+    manifest_path = path.with_suffix(".json")
+    with open(manifest_path, "w") as fd:
+        json.dump(experiment_to_manifest(Experiment.load(path)), fd, indent=2, sort_keys=True)
+    return manifest_path
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Save a file of synthetic data in the version of the ParetoBench file format the library currently "
+            "writes."
+        )
+    )
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR, help="directory to save the file into")
+    parser.add_argument("--force", action="store_true", help="overwrite an existing file for this version")
+    parser.add_argument("--manifest", type=Path, help="rewrite the manifest of an existing file and exit")
+    args = parser.parse_args()
+
+    # Refresh the manifest of a file which already exists (used to bootstrap files saved by older versions)
+    if args.manifest is not None:
+        print(f"Wrote {write_manifest(args.manifest)}")
+        return
+
+    # Save the data, then name the file after the version which actually ended up in it
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    tmp_path = args.out_dir / "_generate_file_version_data.h5"
+    make_experiment().save(tmp_path)
+    with h5py.File(tmp_path) as fd:
+        version = fd.attrs["file_version"]
+    path = args.out_dir / FILENAME_FMT.format(version=version)
+
+    # Regenerating the file of an already released version would destroy the record of how that version wrote data
+    if path.exists() and not args.force:
+        tmp_path.unlink()
+        raise SystemExit(
+            f"{path} already exists, refusing to regenerate the fixture of a released format version. "
+            "Pass --force to overwrite it anyway."
+        )
+
+    shutil.move(tmp_path, path)
+    print(f"Wrote {path} (file version {version})")
+    print(f"Wrote {write_manifest(path)}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_containers.py

@@ -3,38 +3,10 @@
 import os
 import pytest
 import tempfile
-from pathlib import Path
 
-from paretobench.analyze_metrics import normalize_problem_name
 from paretobench.containers import Experiment, Population, History
 
 
-def get_test_files():
-    test_dir = Path(__file__).parent / "legacy_file_formats"
-    return [f for f in test_dir.glob("*.h5")]
-
-
-@pytest.mark.parametrize("test_file", get_test_files())
-def test_load_legacy_files(test_file):
-    """
-    Test loading different versions of saved experiment files for backwards compatibility.
-    """
-    exp = Experiment.load(test_file)
-
-    # Some basic checks
-    assert len(exp.runs) == 6
-    for run in exp.runs:
-        assert len(run) == 8
-        assert len(run.reports[0]) == 50
-        assert run.reports[0].m == 2
-        assert run.reports[0].n == 5
-        assert run.reports[0].g.shape[1] == 0
-
-    # Check problems are right
-    probs = set(normalize_problem_name(x.problem) for x in exp.runs)
-    assert probs == {"ZDT2 (n=5)", "ZDT4 (n=5)", "ZDT6 (n=5)"}
-
-
 @pytest.mark.parametrize("generate_names", [False, True])
 def test_experiment_save_load(generate_names):
     """