diff --git a/.factory/benchmarks/forecastbench.toml b/.factory/benchmarks/forecastbench.toml new file mode 100644 index 0000000..fc732a9 --- /dev/null +++ b/.factory/benchmarks/forecastbench.toml @@ -0,0 +1,17 @@ +[meta] +name = "forecastbench" +description = "ForecastBench — Brier Index gate with graded pytest ladder" + +[test] +format = "pytest" +command = "uv run pytest gate/ -q" +timeout = 600 + +[instances] +format = "directory" + +[seed_workflow] +name = "" + +[scoring] +method = "partial_credit" diff --git a/.gitignore b/.gitignore index 5f70a89..217fb6a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,8 @@ build/ .pytest_cache/ *.egg .venv/ -.factory +.factory/* +!.factory/benchmarks/ skills/ submissions/ response_logs/