feat: builder-only seed + mixed lv1/lv2 calibration for outer loop evolution

.factory/outer_loop/calibration.json

@@ -0,0 +1,72 @@
+{
+  "instances": {
+    "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 518.5
+    },
+    "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 242.2
+    },
+    "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 307.5
+    },
+    "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 319.5
+    },
+    "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 1006.9
+    },
+    "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 410.6
+    },
+    "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 31.4
+    },
+    "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 235.8
+    },
+    "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 79.2
+    },
+    "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1": {
+      "score": 1.0,
+      "resolved": true,
+      "elapsed_seconds": 49.6
+    }
+  },
+  "training": [
+    "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1",
+    "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1",
+    "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1",
+    "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1",
+    "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1",
+    "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1",
+    "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1"
+  ],
+  "holdout": [
+    "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1",
+    "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1",
+    "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1"
+  ],
+  "total": 10,
+  "seed_score": 1.0,
+  "resolved_count": 10,
+  "total_elapsed_seconds": 3201.2
+}

.factory/outer_loop/calibration_lv2.json

@@ -0,0 +1,74 @@
+{
+  "instances": {
+    "astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 259.2
+    },
+    "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 267.7
+    },
+    "huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 692.2
+    },
+    "lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 1888.0
+    },
+    "mlflow__mlflow.93dab383.test_config.c63d41b0.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 160.4
+    },
+    "mwaskom__seaborn.7001ebe7.test_regression.ce8c62e2.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 401.3
+    },
+    "pandas-dev__pandas.82fa2715.test_col.a592871d.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 394.0
+    },
+    "pydata__xarray.97f3a746.test_coordinate_transform.6cacb660.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 548.0
+    },
+    "sympy__sympy.c1097516.test_puiseux.cd575f09.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 1444.3
+    },
+    "mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2": {
+      "score": 0.0,
+      "resolved": false,
+      "elapsed_seconds": 943.0
+    }
+  },
+  "total": 10,
+  "seed_score": 0.0,
+  "seed_name": "featurebench-builder-only",
+  "resolved_count": 0,
+  "total_elapsed_seconds": 6998.1,
+  "level": "lv2",
+  "training": [
+    "astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2",
+    "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2",
+    "huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2",
+    "lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2",
+    "mlflow__mlflow.93dab383.test_config.c63d41b0.lv2",
+    "mwaskom__seaborn.7001ebe7.test_regression.ce8c62e2.lv2",
+    "pandas-dev__pandas.82fa2715.test_col.a592871d.lv2"
+  ],
+  "holdout": [
+    "pydata__xarray.97f3a746.test_coordinate_transform.6cacb660.lv2",
+    "sympy__sympy.c1097516.test_puiseux.cd575f09.lv2",
+    "mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2"
+  ]
+}

.factory/outer_loop/progress.jsonl

@@ -0,0 +1,84 @@
+{"event_type": "calibration_start", "instances": 10, "timestamp": "2026-08-15T22:16:43"}
+{"event_type": "cal_instance_start", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "index": 1, "timestamp": "2026-08-15T22:16:43"}
+{"event_type": "cal_instance_done", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "score": 1.0, "elapsed": 518.5, "timestamp": "2026-08-15T22:25:22"}
+{"event_type": "cal_instance_start", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1", "index": 2, "timestamp": "2026-08-15T22:25:22"}
+{"event_type": "cal_instance_done", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1", "score": 1.0, "elapsed": 242.2, "timestamp": "2026-08-15T22:29:24"}
+{"event_type": "cal_instance_start", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1", "index": 3, "timestamp": "2026-08-15T22:29:24"}
+{"event_type": "cal_instance_done", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1", "score": 1.0, "elapsed": 307.5, "timestamp": "2026-08-15T22:34:31"}
+{"event_type": "cal_instance_start", "instance": "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1", "index": 4, "timestamp": "2026-08-15T22:34:31"}
+{"event_type": "cal_instance_done", "instance": "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1", "score": 1.0, "elapsed": 319.5, "timestamp": "2026-08-15T22:39:51"}
+{"event_type": "cal_instance_start", "instance": "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1", "index": 5, "timestamp": "2026-08-15T22:39:51"}
+{"event_type": "cal_instance_done", "instance": "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1", "score": 1.0, "elapsed": 1006.9, "timestamp": "2026-08-15T22:56:38"}
+{"event_type": "cal_instance_start", "instance": "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1", "index": 6, "timestamp": "2026-08-15T22:56:38"}
+{"event_type": "cal_instance_done", "instance": "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1", "score": 1.0, "elapsed": 410.6, "timestamp": "2026-08-15T23:03:28"}
+{"event_type": "cal_instance_start", "instance": "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1", "index": 7, "timestamp": "2026-08-15T23:03:28"}
+{"event_type": "cal_instance_start", "instance": "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1", "index": 7, "timestamp": "2026-08-15T23:05:40"}
+{"event_type": "cal_instance_done", "instance": "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1", "score": 1.0, "elapsed": 31.4, "timestamp": "2026-08-15T23:06:12"}
+{"event_type": "cal_instance_start", "instance": "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1", "index": 8, "timestamp": "2026-08-15T23:06:12"}
+{"event_type": "cal_instance_done", "instance": "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1", "score": 1.0, "elapsed": 235.8, "timestamp": "2026-08-15T23:10:07"}
+{"event_type": "cal_instance_start", "instance": "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1", "index": 9, "timestamp": "2026-08-15T23:10:07"}
+{"event_type": "cal_instance_done", "instance": "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1", "score": 1.0, "elapsed": 79.2, "timestamp": "2026-08-15T23:11:27"}
+{"event_type": "cal_instance_start", "instance": "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1", "index": 10, "timestamp": "2026-08-15T23:11:27"}
+{"event_type": "cal_instance_done", "instance": "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1", "score": 1.0, "elapsed": 49.6, "timestamp": "2026-08-15T23:12:16"}
+{"event_type": "calibration_complete", "seed_score": 1.0, "resolved": 10, "total": 10, "training": 7, "holdout": 3, "timestamp": "2026-08-15T23:12:16"}
+{"event_type": "generation_start", "generation": 0, "budget_remaining": 20, "timestamp": "2026-08-15T23:12:58"}
+{"event_type": "generation_start", "generation": 0, "budget_remaining": 20, "timestamp": "2026-08-15T23:20:58"}
+{"event_type": "calibration_v2_start", "instances": 10, "seed": "featurebench-builder-only", "timestamp": "2026-08-16T00:53:59"}
+{"event_type": "cal_v2_instance_start", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "index": 1, "timestamp": "2026-08-16T00:53:59"}
+{"event_type": "cal_v2_instance_done", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "score": 1.0, "elapsed": 336.7, "timestamp": "2026-08-16T00:59:36"}
+{"event_type": "cal_v2_instance_start", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1", "index": 2, "timestamp": "2026-08-16T00:59:36"}
+{"event_type": "calibration_v2_start", "instances": 10, "seed": "featurebench-builder-only", "timestamp": "2026-08-16T01:01:25"}
+{"event_type": "cal_v2_instance_start", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "index": 1, "timestamp": "2026-08-16T01:01:25"}
+{"event_type": "cal_v2_instance_done", "instance": "pydantic__pydantic.e1dcaf9e.test_deprecated_fields.40a2ec54.lv1", "score": 1.0, "elapsed": 421.2, "timestamp": "2026-08-16T01:08:26"}
+{"event_type": "cal_v2_instance_start", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1", "index": 2, "timestamp": "2026-08-16T01:08:26"}
+{"event_type": "cal_v2_instance_done", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1", "score": 1.0, "elapsed": 110.6, "timestamp": "2026-08-16T01:10:17"}
+{"event_type": "cal_v2_instance_start", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1", "index": 3, "timestamp": "2026-08-16T01:10:17"}
+{"event_type": "cal_v2_instance_done", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv1", "score": 1.0, "elapsed": 338.2, "timestamp": "2026-08-16T01:15:55"}
+{"event_type": "cal_v2_instance_start", "instance": "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1", "index": 4, "timestamp": "2026-08-16T01:15:55"}
+{"event_type": "cal_v2_instance_done", "instance": "mwaskom__seaborn.7001ebe7.test_bar.123ed709.lv1", "score": 1.0, "elapsed": 201.9, "timestamp": "2026-08-16T01:19:17"}
+{"event_type": "cal_v2_instance_start", "instance": "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1", "index": 5, "timestamp": "2026-08-16T01:19:17"}
+{"event_type": "cal_v2_instance_done", "instance": "sphinx-doc__sphinx.e347e59c.test_build_gettext.2721e644.lv1", "score": 1.0, "elapsed": 217.9, "timestamp": "2026-08-16T01:22:55"}
+{"event_type": "cal_v2_instance_start", "instance": "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1", "index": 6, "timestamp": "2026-08-16T01:22:55"}
+{"event_type": "cal_v2_instance_done", "instance": "matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1", "score": 1.0, "elapsed": 243.7, "timestamp": "2026-08-16T01:26:58"}
+{"event_type": "cal_v2_instance_start", "instance": "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1", "index": 7, "timestamp": "2026-08-16T01:26:58"}
+{"event_type": "calibration_lv2_start", "instances": 10, "seed": "featurebench-builder-only", "level": "lv2", "timestamp": "2026-08-16T01:27:27"}
+{"event_type": "cal_lv2_instance_start", "instance": "astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2", "index": 1, "timestamp": "2026-08-16T01:27:27"}
+{"event_type": "cal_lv2_instance_done", "instance": "astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2", "score": 0.0, "elapsed": 259.2, "timestamp": "2026-08-16T01:31:46"}
+{"event_type": "cal_lv2_instance_start", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2", "index": 2, "timestamp": "2026-08-16T01:31:46"}
+{"event_type": "cal_lv2_instance_done", "instance": "fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2", "score": 0.0, "elapsed": 267.7, "timestamp": "2026-08-16T01:36:14"}
+{"event_type": "cal_lv2_instance_start", "instance": "huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2", "index": 3, "timestamp": "2026-08-16T01:36:14"}
+{"event_type": "cal_v2_instance_done", "instance": "sympy__sympy.c1097516.test_inverse.c240ffe7.lv1", "score": 1.0, "elapsed": 1024.0, "timestamp": "2026-08-16T01:44:02"}
+{"event_type": "cal_v2_instance_start", "instance": "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1", "index": 8, "timestamp": "2026-08-16T01:44:02"}
+{"event_type": "cal_lv2_instance_done", "instance": "huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2", "score": 0.0, "elapsed": 692.2, "timestamp": "2026-08-16T01:47:46"}
+{"event_type": "cal_lv2_instance_start", "instance": "lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2", "index": 4, "timestamp": "2026-08-16T01:47:46"}
+{"event_type": "cal_v2_instance_done", "instance": "mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1", "score": 1.0, "elapsed": 427.2, "timestamp": "2026-08-16T01:51:10"}
+{"event_type": "cal_v2_instance_start", "instance": "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1", "index": 9, "timestamp": "2026-08-16T01:51:10"}
+{"event_type": "cal_lv2_instance_start", "instance": "lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2", "index": 4, "timestamp": "2026-08-16T01:52:56"}
+{"event_type": "cal_v2_instance_done", "instance": "pytest-dev__pytest.68016f0e.raises_group.c28bf36a.lv1", "score": 1.0, "elapsed": 550.4, "timestamp": "2026-08-16T02:00:20"}
+{"event_type": "cal_v2_instance_start", "instance": "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1", "index": 10, "timestamp": "2026-08-16T02:00:20"}
+{"event_type": "cal_v2_instance_done", "instance": "pypa__packaging.013f3b03.test_metadata.e00b5801.lv1", "score": 1.0, "elapsed": 413.4, "timestamp": "2026-08-16T02:07:13"}
+{"event_type": "calibration_v2_complete", "seed_score": 1.0, "resolved": 10, "total": 10, "training": 7, "holdout": 3, "timestamp": "2026-08-16T02:07:13"}
+{"event_type": "cal_lv2_instance_done", "instance": "lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2", "score": 0.0, "elapsed": 1888.0, "timestamp": "2026-08-16T02:24:24"}
+{"event_type": "cal_lv2_instance_start", "instance": "mlflow__mlflow.93dab383.test_config.c63d41b0.lv2", "index": 5, "timestamp": "2026-08-16T02:24:24"}
+{"event_type": "cal_lv2_instance_done", "instance": "mlflow__mlflow.93dab383.test_config.c63d41b0.lv2", "score": 0.0, "elapsed": 160.4, "timestamp": "2026-08-16T02:27:05"}
+{"event_type": "cal_lv2_instance_start", "instance": "mwaskom__seaborn.7001ebe7.test_regression.ce8c62e2.lv2", "index": 6, "timestamp": "2026-08-16T02:27:05"}
+{"event_type": "cal_lv2_instance_done", "instance": "mwaskom__seaborn.7001ebe7.test_regression.ce8c62e2.lv2", "score": 0.0, "elapsed": 401.3, "timestamp": "2026-08-16T02:33:46"}
+{"event_type": "cal_lv2_instance_start", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv2", "index": 7, "timestamp": "2026-08-16T02:33:46"}
+{"event_type": "cal_lv2_instance_done", "instance": "pandas-dev__pandas.82fa2715.test_col.a592871d.lv2", "score": 0.0, "elapsed": 394.0, "timestamp": "2026-08-16T02:40:20"}
+{"event_type": "cal_lv2_instance_start", "instance": "pydata__xarray.97f3a746.test_coordinate_transform.6cacb660.lv2", "index": 8, "timestamp": "2026-08-16T02:40:20"}
+{"event_type": "cal_lv2_instance_done", "instance": "pydata__xarray.97f3a746.test_coordinate_transform.6cacb660.lv2", "score": 0.0, "elapsed": 548.0, "timestamp": "2026-08-16T02:49:28"}
+{"event_type": "cal_lv2_instance_start", "instance": "sympy__sympy.c1097516.test_puiseux.cd575f09.lv2", "index": 9, "timestamp": "2026-08-16T02:49:28"}
+{"event_type": "cal_lv2_instance_done", "instance": "sympy__sympy.c1097516.test_puiseux.cd575f09.lv2", "score": 0.0, "elapsed": 1444.3, "timestamp": "2026-08-16T03:13:33"}
+{"event_type": "cal_lv2_instance_start", "instance": "mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2", "index": 10, "timestamp": "2026-08-16T03:13:33"}
+{"event_type": "cal_lv2_instance_done", "instance": "mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2", "score": 0.0, "elapsed": 943.0, "timestamp": "2026-08-16T03:29:16"}
+{"event_type": "calibration_lv2_complete", "seed_score": 0.0, "resolved": 0, "total": 10, "training_count": 7, "holdout_count": 3, "timestamp": "2026-08-16T03:29:16"}
+{"event_type": "evolution_lv2_start", "training": 7, "holdout": 3, "population": 4, "budget": 20, "generations": 2, "timestamp": "2026-08-16T03:30:40"}
+{"event_type": "generation_start", "generation": 0, "budget_remaining": 20, "timestamp": "2026-08-16T03:30:40"}
+{"event_type": "generation_start", "generation": 0, "budget_remaining": 20, "timestamp": "2026-08-16T03:33:07"}
+{"event_type": "generation_start", "generation": 0, "budget_remaining": 20, "timestamp": "2026-08-16T03:49:13"}
+{"event_type": "generation_complete", "generation": 0, "best_score": 0.0, "mean_score": 0.0, "duration_seconds": 25611.55, "timestamp": "2026-08-16T10:37:32"}
+{"event_type": "checkpoint_saved", "generation": 0, "path": "/Users/akasriva/cursor-projects/remote-factory/.factory-worktrees/run-17122260/.factory/outer_loop/checkpoint_gen_0.json", "timestamp": "2026-08-16T10:37:32"}
+{"event_type": "generation_start", "generation": 1, "budget_remaining": 15, "timestamp": "2026-08-16T10:37:32"}
+{"event_type": "generation_complete", "generation": 1, "best_score": 0.0, "mean_score": 0.0, "duration_seconds": 13305.75, "timestamp": "2026-08-16T14:19:18"}
+{"event_type": "checkpoint_saved", "generation": 1, "path": "/Users/akasriva/cursor-projects/remote-factory/.factory-worktrees/run-17122260/.factory/outer_loop/checkpoint_gen_1.json", "timestamp": "2026-08-16T14:19:18"}
+{"event_type": "generation_start", "generation": 2, "budget_remaining": 9, "timestamp": "2026-08-16T14:19:18"}

factory/cli/__init__.py

@@ -39,6 +39,7 @@
     cmd_tmux_stop as cmd_tmux_stop,
 )
 from factory.cli.mempalace import cmd_mempalace as cmd_mempalace
+from factory.cli.outer_loop import cmd_outer_loop as cmd_outer_loop
 from factory.cli.contained import (
     cmd_contained as cmd_contained,
 )

factory/cli/_helpers.py

@@ -39,6 +39,7 @@
     "frontend-design-scan",
     "evolve",
     "deep-research",
+    "outer-loop",
 ]
 
 
@@ -55,6 +56,7 @@
     "study",
     "swebench",
     "frontend-design-scan",
+    "outer-loop",
 ]