From 4d36e0139f2d285d12df3c2728f06e912681b813 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 09:23:08 -0400 Subject: [PATCH 1/3] Configure outer loop with correct forecastbench benchmark - Create .factory/benchmarks/forecastbench.toml with pytest format and gate test command (shadows incorrect built-in JSON config) - Fix .factory/outer_loop/config.json benchmark from "featurebench" to "forecastbench" - Add .factory/scripts/trim-backlog.sh workaround for remote-factory #1338 (--focus gap) Addresses #161 Co-Authored-By: Claude Opus 4.6 (1M context) --- .factory/benchmarks/forecastbench.toml | 17 +++++++++++++++ .factory/outer_loop/config.json | 26 ++++++++++++++++++++++ .factory/scripts/trim-backlog.sh | 30 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 .factory/benchmarks/forecastbench.toml create mode 100644 .factory/outer_loop/config.json create mode 100755 .factory/scripts/trim-backlog.sh diff --git a/.factory/benchmarks/forecastbench.toml b/.factory/benchmarks/forecastbench.toml new file mode 100644 index 0000000..fc732a9 --- /dev/null +++ b/.factory/benchmarks/forecastbench.toml @@ -0,0 +1,17 @@ +[meta] +name = "forecastbench" +description = "ForecastBench — Brier Index gate with graded pytest ladder" + +[test] +format = "pytest" +command = "uv run pytest gate/ -q" +timeout = 600 + +[instances] +format = "directory" + +[seed_workflow] +name = "" + +[scoring] +method = "partial_credit" diff --git a/.factory/outer_loop/config.json b/.factory/outer_loop/config.json new file mode 100644 index 0000000..b17a3e4 --- /dev/null +++ b/.factory/outer_loop/config.json @@ -0,0 +1,26 @@ +{ + "benchmark": "forecastbench", + "budget": 9, + "population_size": 3, + "tournament_size": 3, + "mutation_rate": 0.3, + "target_score": null, + "frozen_node_ids": [], + "mandatory_node_roles": [], + "feature_axes": [ + "depth", + "fork_degree", + "agent_count", + "gate_count" + ], + "mutation_strategy": "weighted_random", + "designer_count": 0, + "training_instances": [], + "holdout_instances": [], + "plateau_window": 3, + "plateau_threshold": 0.01, + "diversity_floor": 0.2, + "target_project": "", + "test_command": "uv run pytest gate/ -q", + "early_stop_unchanged": 3 +} \ No newline at end of file diff --git a/.factory/scripts/trim-backlog.sh b/.factory/scripts/trim-backlog.sh new file mode 100755 index 0000000..274f19a --- /dev/null +++ b/.factory/scripts/trim-backlog.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: $0 ''" + echo "Trims backlog to a single item (saves backup first)." + exit 1 +fi + +BACKLOG=".factory/strategy/backlog.md" +BACKUP=".factory/strategy/backlog-full.md" + +if [ ! -f "$BACKLOG" ]; then + echo "ERROR: $BACKLOG not found" + exit 1 +fi + +cp "$BACKLOG" "$BACKUP" +echo "Saved backup to $BACKUP" + +cat > "$BACKLOG" < Date: Thu, 20 Aug 2026 09:49:45 -0400 Subject: [PATCH 2/3] Track only benchmark TOML; remove runtime files from git Add !.factory/benchmarks/ exclusion to .gitignore so the benchmark config is tracked while the rest of .factory stays ignored. Remove config.json and trim-backlog.sh from tracking (runtime/local-only). Co-Authored-By: Claude Opus 4.6 (1M context) --- .factory/outer_loop/config.json | 26 -------------------------- .factory/scripts/trim-backlog.sh | 30 ------------------------------ .gitignore | 1 + 3 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 .factory/outer_loop/config.json delete mode 100755 .factory/scripts/trim-backlog.sh diff --git a/.factory/outer_loop/config.json b/.factory/outer_loop/config.json deleted file mode 100644 index b17a3e4..0000000 --- a/.factory/outer_loop/config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "benchmark": "forecastbench", - "budget": 9, - "population_size": 3, - "tournament_size": 3, - "mutation_rate": 0.3, - "target_score": null, - "frozen_node_ids": [], - "mandatory_node_roles": [], - "feature_axes": [ - "depth", - "fork_degree", - "agent_count", - "gate_count" - ], - "mutation_strategy": "weighted_random", - "designer_count": 0, - "training_instances": [], - "holdout_instances": [], - "plateau_window": 3, - "plateau_threshold": 0.01, - "diversity_floor": 0.2, - "target_project": "", - "test_command": "uv run pytest gate/ -q", - "early_stop_unchanged": 3 -} \ No newline at end of file diff --git a/.factory/scripts/trim-backlog.sh b/.factory/scripts/trim-backlog.sh deleted file mode 100755 index 274f19a..0000000 --- a/.factory/scripts/trim-backlog.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ $# -ne 1 ]; then - echo "Usage: $0 ''" - echo "Trims backlog to a single item (saves backup first)." - exit 1 -fi - -BACKLOG=".factory/strategy/backlog.md" -BACKUP=".factory/strategy/backlog-full.md" - -if [ ! -f "$BACKLOG" ]; then - echo "ERROR: $BACKLOG not found" - exit 1 -fi - -cp "$BACKLOG" "$BACKUP" -echo "Saved backup to $BACKUP" - -cat > "$BACKLOG" < Date: Thu, 20 Aug 2026 11:24:37 -0400 Subject: [PATCH 3/3] Fix gitignore negation so new benchmark files are tracked Change .factory to .factory/* so git descends into the directory and evaluates the !.factory/benchmarks/ negation pattern for new files. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b2a63d3..217fb6a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ build/ .pytest_cache/ *.egg .venv/ -.factory +.factory/* !.factory/benchmarks/ skills/ submissions/